async-bundles (!117)

This patch contains the following:

- Initial async VFS code (for reading / writing files)
- Simplified Http Client code
- Added async support to Http Client, RequestHandler
- Improved RequestHandler logging
- Deferred bundle loading to EasyAssetPatch
- Make GetManifestJson run async

This comes with a number of benefits:

- When downloading bundles, it will mention which files succeeded or failed to download
- Bundle loading happens in the initial screen, not the white screen
- Fixed the issue where bundle loading could break bepinex loading (too long load time)
- Modders can now make async http request and read/write files async

I removed logging of sessionid inside the RequestHandler for each request, sessionid is already visible from bepinex log startup parameters.

At last, sorry for the amount of commits it took. I initially wanted to target the 3.9.0 branch, but decided to use 3.8.1 instead as async request can really help out some mods.

Reviewed-on: SPT-AKI/Modules#117
Co-authored-by: Merijn Hendriks <merijn.d.hendriks@gmail.com>
Co-committed-by: Merijn Hendriks <merijn.d.hendriks@gmail.com>

project/Aki.Common/Http/Client.cs

@@ -1,14 +1,14 @@
 using System;
 using System.IO;
 using System.Net.Http;
-using Aki.Common.Http;
+using System.Text;
+using System.Threading.Tasks;
 using Aki.Common.Utils;
 
 namespace Aki.Common.Http
 {
-    // NOTE: you do not want to dispose this, keep a reference for the lifetime
+    // NOTE: Don't dispose this, keep a reference for the lifetime of the
-    //       of the application.
+    //       application.
-    // NOTE: cannot be made async due to Unity's limitations.
     public class Client : IDisposable
     {
         protected readonly HttpClient _httpv;
@@ -22,9 +22,9 @@ namespace Aki.Common.Http
             _accountId = accountId;
             _retries = retries;
 
-            var handler = new HttpClientHandler()
+            var handler = new HttpClientHandler
             {
-                // force setting cookies in header instead of CookieContainer
+                // set cookies in header instead
                 UseCookies = false
             };
 
@@ -43,7 +43,7 @@ namespace Aki.Common.Http
             };
         }
 
-        protected byte[] Send(HttpMethod method, string path, byte[] data, bool compress = true)
+        protected async Task<byte[]> SendAsync(HttpMethod method, string path, byte[] data, bool zipped = true)
         {
             HttpResponseMessage response = null;
 
@@ -51,17 +51,17 @@ namespace Aki.Common.Http
             {
                 if (data != null)
                 {
-                    // if there is data, convert to payload
-                    byte[] payload = (compress)
-                        ? Zlib.Compress(data, ZlibCompression.Maximum)
-                        : data;
-
                     // add payload to request
-                    request.Content = new ByteArrayContent(payload);
+                    if (zipped)
+                    {
+                        data = Zlib.Compress(data, ZlibCompression.Maximum);
+                    }
+
+                    request.Content = new ByteArrayContent(data);
                 }
 
                 // send request
-                response = _httpv.SendAsync(request).Result;
+                response = await _httpv.SendAsync(request);
             }
 
             if (!response.IsSuccessStatusCode)
@@ -72,85 +72,79 @@ namespace Aki.Common.Http
 
             using (var ms = new MemoryStream())
             {
-                using (var stream = response.Content.ReadAsStreamAsync().Result)
+                using (var stream = await response.Content.ReadAsStreamAsync())
                 {
                     // grap response payload
-                    stream.CopyTo(ms);
+                    await stream.CopyToAsync(ms);
-                    var bytes = ms.ToArray();
+                    var body = ms.ToArray();
 
-                    if (bytes != null)
+                    if (Zlib.IsCompressed(body))
                     {
-                        // payload contains data
+                        body = Zlib.Decompress(body);
-                        return Zlib.IsCompressed(bytes)
-                            ? Zlib.Decompress(bytes)
-                            : bytes;
                     }
+
+                    if (body == null)
+                    {
+                        // payload doesn't contains data
+                        var code = response.StatusCode.ToString();
+                        body = Encoding.UTF8.GetBytes(code);
+                    }
+
+                    return body;
+                }
+            }
+        }
+
+        protected async Task<byte[]> SendWithRetriesAsync(HttpMethod method, string path, byte[] data, bool compress = true)
+        {
+            var error = new Exception("Internal error");
+
+            // NOTE: <= is intentional. 0 is send, 1/2/3 is retry
+            for (var i = 0; i <= _retries; ++i)
+            {
+                try
+                {
+                    return await SendAsync(method, path, data, compress);
+                }
+                catch (Exception ex)
+                {
+                    error = ex;
                 }
             }
 
-            // response returned no data
+            throw error;
-            return null;
+        }
+
+        public async Task<byte[]> GetAsync(string path)
+        {
+            return await SendWithRetriesAsync(HttpMethod.Get, path, null);
         }
 
         public byte[] Get(string path)
         {
-            var error = new Exception("Internal error");
+            return Task.Run(() => GetAsync(path)).Result;
-
-            // NOTE: <= is intentional, 0 is send, 1,2,3 is retry
-            for (var i = 0; i <= _retries; ++i)
-            {
-                try
-                {
-                    return Send(HttpMethod.Get, path, null, false);
-                }
-                catch (Exception ex)
-                {
-                    error = ex;
-                }
-            }
-
-            throw error;
         }
 
-        public byte[] Post(string path, byte[] data, bool compressed = true)
+        public async Task<byte[]> PostAsync(string path, byte[] data, bool compress = true)
         {
-            var error = new Exception("Internal error");
+            return await SendWithRetriesAsync(HttpMethod.Post, path, data, compress);
-
-            // NOTE: <= is intentional, 0 is send, 1,2,3 is retry
-            for (var i = 0; i <= _retries; ++i)
-            {
-                try
-                {
-                    return Send(HttpMethod.Post, path, data, compressed);
-                }
-                catch (Exception ex)
-                {
-                    error = ex;
-                }
-            }
-
-            throw error;
         }
 
-        public void Put(string path, byte[] data, bool compressed = true)
+        public byte[] Post(string path, byte[] data, bool compress = true)
         {
-            var error = new Exception("Internal error");
+            return Task.Run(() => PostAsync(path, data, compress)).Result;
+        }
 
-            // NOTE: <= is intentional, 0 is send, 1,2,3 is retry
+        // NOTE: returns status code as bytes
-            for (var i = 0; i <= _retries; ++i)
+        public async Task<byte[]> PutAsync(string path, byte[] data, bool compress = true)
-            {
+        {
-                try
+            return await SendWithRetriesAsync(HttpMethod.Post, path, data, compress);
-                {
+        }
-                    Send(HttpMethod.Put, path, data, compressed);
-                    return;
-                }
-                catch (Exception ex)
-                {
-                    error = ex;
-                }
-            }
 
-            throw error;
+        // NOTE: returns status code as bytes
+        public byte[] Put(string path, byte[] data, bool compress = true)
+        {
+            return Task.Run(() => PutAsync(path, data, compress)).Result;
         }
 
         public void Dispose()

project/Aki.Common/Http/RequestHandler.cs

@@ -43,65 +43,91 @@ namespace Aki.Common.Http
             HttpClient = new Client(Host, SessionId);
         }
 
-        private static void ValidateData(byte[] data)
+        private static void ValidateData(string path, byte[] data)
         {
             if (data == null)
             {
-                _logger.LogError($"Request failed, body is null");
+                _logger.LogError($"[REQUEST FAILED] {path}");
             }
 
-            _logger.LogInfo($"Request was successful");
+            _logger.LogInfo($"[REQUEST SUCCESSFUL] {path}");
         }
 
-        private static void ValidateJson(string json)
+        private static void ValidateJson(string path, string json)
         {
             if (string.IsNullOrWhiteSpace(json))
             {
-                _logger.LogError($"Request failed, body is null");
+                _logger.LogError($"[REQUEST FAILED] {path}");
             }
 
-            _logger.LogInfo($"Request was successful");
+            _logger.LogInfo($"[REQUEST SUCCESSFUL] {path}");
+        }
+
+        public static async Task<byte[]> GetDataAsync(string path)
+        {
+            _logger.LogInfo($"[REQUEST]: {path}");
+            
+            var data = await HttpClient.GetAsync(path);
+
+            ValidateData(path, data);
+            return data;
         }
 
         public static byte[] GetData(string path)
         {
-            _logger.LogInfo($"Request GET data: {SessionId}:{path}");
+            return Task.Run(() => GetData(path)).Result;
-            
+        }
-            var data = HttpClient.Get(path);
 
-            ValidateData(data);
+        public static async Task<string> GetJsonAsync(string path)
-            return data;
+        {
+            _logger.LogInfo($"[REQUEST]: {path}");
+            
+            var payload = await HttpClient.GetAsync(path);
+            var body = Encoding.UTF8.GetString(payload);
+
+            ValidateJson(path, body);
+            return body;
         }
 
         public static string GetJson(string path)
         {
-            _logger.LogInfo($"Request GET json: {SessionId}:{path}");
+            return Task.Run(() => GetJsonAsync(path)).Result;
-            
-            var payload = HttpClient.Get(path);
-            var body = Encoding.UTF8.GetString(payload);
-
-            ValidateJson(body);
-            return body;
         }
 
-        public static string PostJson(string path, string json)
+        public static string PostJsonAsync(string path, string json)
         {
-            _logger.LogInfo($"Request POST json: {SessionId}:{path}");
+            _logger.LogInfo($"[REQUEST]: {path}");
             
             var payload = Encoding.UTF8.GetBytes(json);
             var data = HttpClient.Post(path, payload);
             var body = Encoding.UTF8.GetString(data);
 
-            ValidateJson(body);
+            ValidateJson(path, body);
             return body;
         }
 
-        public static void PutJson(string path, string json)
+        public static string PostJson(string path, string json)
         {
-            _logger.LogInfo($"Request PUT json: {SessionId}:{path}");
+            return Task.Run(() => PostJsonAsync(path, json)).Result;
+        }
+
+        // NOTE: returns status code
+        public static async Task<string> PutJsonAsync(string path, string json)
+        {
+            _logger.LogInfo($"[REQUEST]: {path}");
 
             var payload = Encoding.UTF8.GetBytes(json);
-            HttpClient.Put(path, payload);
+            var data = await HttpClient.PutAsync(path, payload);
+            var body = Encoding.UTF8.GetString(data);
+
+            ValidateJson(path, body);
+            return body;
+        }
+
+        // NOTE: returns status code
+        public static string PutJson(string path, string json)
+        {
+            return Task.Run(() => PutJsonAsync(path, json)).Result;
         }
 
 #region DEPRECATED, REMOVE IN 3.8.1
@@ -120,7 +146,7 @@ namespace Aki.Common.Http
             var request = new Request();
             var data = request.Send(url, "GET", null, headers: headers);
 
-            ValidateData(data);
+            ValidateData(url, data);
             return data;
 
         }
@@ -141,7 +167,7 @@ namespace Aki.Common.Http
             var data = request.Send(url, "GET", headers: headers);
             var body = Encoding.UTF8.GetString(data);
 
-            ValidateJson(body);
+            ValidateJson(url, body);
             return body;
 
         }
@@ -164,7 +190,7 @@ namespace Aki.Common.Http
             var data = request.Send(url, "POST", payload, true, mime, headers);
             var body = Encoding.UTF8.GetString(data);
 
-            ValidateJson(body);
+            ValidateJson(url, body);
             return body;
 
         }

project/Aki.Common/Utils/VFS.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Threading.Tasks;
 
 namespace Aki.Common.Utils
 {
@@ -97,6 +98,22 @@ namespace Aki.Common.Utils
             return File.ReadAllBytes(filepath);
         }
 
+        /// <summary>
+        /// Get file content as bytes.
+        /// </summary>
+        public static async Task<byte[]> ReadFileAsync(string filepath)
+        {
+            byte[] result;
+
+            using (var fs = File.Open(filepath, FileMode.Open))
+            {
+                result = new byte[fs.Length];
+                await fs.ReadAsync(result, 0, (int)fs.Length);
+            }
+
+            return result;
+        }
+
         /// <summary>
         /// Get file content as string.
         /// </summary>
@@ -105,6 +122,20 @@ namespace Aki.Common.Utils
             return File.ReadAllText(filepath);
         }
 
+        /// <summary>
+        /// Get file content as string.
+        /// </summary>
+        public static async Task<string> ReadTextFileAsync(string filepath)
+        {
+            using (var fs = File.Open(filepath, FileMode.Open))
+            {
+                using (var sr = new StreamReader(fs))
+                {
+                    return await sr.ReadToEndAsync();
+                }
+            }
+        }
+
         /// <summary>
         /// Write data to file.
         /// </summary>
@@ -118,6 +149,22 @@ namespace Aki.Common.Utils
             File.WriteAllBytes(filepath, data);
         }
 
+        /// <summary>
+        /// Write data to file.
+        /// </summary>
+        public static async Task WriteFileAsync(string filepath, byte[] data)
+        {
+            if (!Exists(filepath))
+            {
+                CreateDirectory(filepath.GetDirectory());
+            }
+
+            using (FileStream stream = File.Open(filepath, FileMode.OpenOrCreate, FileAccess.Write))
+            {
+                await stream.WriteAsync(data, 0, data.Length);
+            }
+        }
+
         /// <summary>
         /// Write string to file.
         /// </summary>

project/Aki.Custom/AkiCustomPlugin.cs

@@ -21,7 +21,6 @@ namespace Aki.Custom
             try
             {
                 // Bundle patches should always load first
-                BundleManager.GetBundles();
                 new EasyAssetsPatch().Enable();
                 new EasyBundlePatch().Enable();
 

project/Aki.Custom/Patches/EasyAssetsPatch.cs

@@ -68,6 +68,12 @@ namespace Aki.Custom.Patches
                 ? await GetManifestJson(jsonfile)
                 : await GetManifestBundle(filepath);
 
+            // lazy-initialize aki bundles
+            if (BundleManager.Bundles.Keys.Count == 0)
+            {
+                await BundleManager.GetBundles();
+            }
+
             // create bundles array from obfuscated type
             var bundleNames = manifest.GetAllAssetBundles()
                 .Union(BundleManager.Bundles.Keys)
@@ -122,7 +128,7 @@ namespace Aki.Custom.Patches
 
         private static async Task<CompatibilityAssetBundleManifest> GetManifestJson(string filepath)
         {
-            var text = VFS.ReadTextFile(filepath);
+            var text = await VFS.ReadTextFileAsync(filepath);
 
             /* we cannot parse directly as <string, BundleDetails>, because...
                     [Error  : Unity Log] JsonSerializationException: Expected string when reading UnityEngine.Hash128 type, got 'StartObject' <>. Path '['assets/content/weapons/animations/simple_animations.bundle'].Hash', line 1, position 176.

project/Aki.Custom/Utils/BundleManager.cs

@@ -10,7 +10,7 @@ namespace Aki.Custom.Utils
 {
     public static class BundleManager
     {
-        private static ManualLogSource _logger;
+        private static readonly ManualLogSource _logger;
         public static readonly ConcurrentDictionary<string, BundleItem> Bundles;
         public static string CachePath;
 
@@ -28,7 +28,7 @@ namespace Aki.Custom.Utils
                 : CachePath + bundle.FileName;
         }
 
-        public static void GetBundles()
+        public static async Task GetBundles()
         {
             // get bundles
             var json = RequestHandler.GetJson("/singleplayer/bundles");
@@ -37,16 +37,16 @@ namespace Aki.Custom.Utils
             // register bundles
             var toDownload = new ConcurrentBag<BundleItem>();
 
-            Parallel.ForEach(bundles, (bundle) =>
+            foreach (var bundle in bundles)
             {
                 Bundles.TryAdd(bundle.FileName, bundle);
 
-                if (ShouldReaquire(bundle))
+                if (await ShouldReaquire(bundle))
                 {
                     // mark for download
                     toDownload.Add(bundle);
                 }
-            });
+            }
 
             if (RequestHandler.IsLocal)
             {
@@ -58,17 +58,17 @@ namespace Aki.Custom.Utils
             {
                 // download bundles
                 // NOTE: assumes bundle keys to be unique
-                Parallel.ForEach(toDownload, (bundle) =>
+                foreach (var bundle in toDownload)
                 {
                     // download bundle
                     var filepath = GetBundlePath(bundle);
-                    var data = RequestHandler.GetData($"/files/bundle/{bundle.FileName}");
+                    var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}");
-                    VFS.WriteFile(filepath, data);
+                    await VFS.WriteFileAsync(filepath, data);
-                });
+                }
             }
         }
 
-        private static bool ShouldReaquire(BundleItem bundle)
+        private static async Task<bool> ShouldReaquire(BundleItem bundle)
         {
             if (RequestHandler.IsLocal)
             {
@@ -82,7 +82,7 @@ namespace Aki.Custom.Utils
             if (VFS.Exists(filepath))
             {
                 // calculate hash
-                var data = VFS.ReadFile(filepath);
+                var data = await VFS.ReadFileAsync(filepath);
                 var crc = Crc32.Compute(data);
 
                 if (crc == bundle.Crc)