Merge 382 into master (!125)

Reviewed-on: SPT-AKI/Modules#125

project/Aki.Common/Http/Client.cs

@@ -1,14 +1,14 @@
 using System;
 using System.IO;
 using System.Net.Http;
-using Aki.Common.Http;
+using System.Text;
+using System.Threading.Tasks;
 using Aki.Common.Utils;
 
 namespace Aki.Common.Http
 {
-    // NOTE: you do not want to dispose this, keep a reference for the lifetime
-    //       of the application.
-    // NOTE: cannot be made async due to Unity's limitations.
+    // NOTE: Don't dispose this, keep a reference for the lifetime of the
+    //       application.
     public class Client : IDisposable
     {
         protected readonly HttpClient _httpv;
@@ -22,9 +22,9 @@ namespace Aki.Common.Http
             _accountId = accountId;
             _retries = retries;
 
-            var handler = new HttpClientHandler()
+            var handler = new HttpClientHandler
             {
-                // force setting cookies in header instead of CookieContainer
+                // set cookies in header instead
                 UseCookies = false
             };
 
@@ -43,7 +43,7 @@ namespace Aki.Common.Http
             };
         }
 
-        protected byte[] Send(HttpMethod method, string path, byte[] data, bool compress = true)
+        protected async Task<byte[]> SendAsync(HttpMethod method, string path, byte[] data, bool zipped = true)
         {
             HttpResponseMessage response = null;
 
@@ -51,17 +51,17 @@ namespace Aki.Common.Http
             {
                 if (data != null)
                 {
-                    // if there is data, convert to payload
-                    byte[] payload = (compress)
-                        ? Zlib.Compress(data, ZlibCompression.Maximum)
-                        : data;
-
                     // add payload to request
-                    request.Content = new ByteArrayContent(payload);
+                    if (zipped)
+                    {
+                        data = Zlib.Compress(data, ZlibCompression.Maximum);
+                    }
+
+                    request.Content = new ByteArrayContent(data);
                 }
 
                 // send request
-                response = _httpv.SendAsync(request).Result;
+                response = await _httpv.SendAsync(request);
             }
 
             if (!response.IsSuccessStatusCode)
@@ -72,85 +72,79 @@ namespace Aki.Common.Http
 
             using (var ms = new MemoryStream())
             {
-                using (var stream = response.Content.ReadAsStreamAsync().Result)
+                using (var stream = await response.Content.ReadAsStreamAsync())
                 {
                     // grap response payload
-                    stream.CopyTo(ms);
-                    var bytes = ms.ToArray();
+                    await stream.CopyToAsync(ms);
+                    var body = ms.ToArray();
 
-                    if (bytes != null)
+                    if (Zlib.IsCompressed(body))
                     {
-                        // payload contains data
-                        return Zlib.IsCompressed(bytes)
-                            ? Zlib.Decompress(bytes)
-                            : bytes;
+                        body = Zlib.Decompress(body);
+                    }
+
+                    if (body == null)
+                    {
+                        // payload doesn't contains data
+                        var code = response.StatusCode.ToString();
+                        body = Encoding.UTF8.GetBytes(code);
+                    }
+
+                    return body;
                 }
             }
         }
 
-            // response returned no data
-            return null;
+        protected async Task<byte[]> SendWithRetriesAsync(HttpMethod method, string path, byte[] data, bool compress = true)
+        {
+            var error = new Exception("Internal error");
+
+            // NOTE: <= is intentional. 0 is send, 1/2/3 is retry
+            for (var i = 0; i <= _retries; ++i)
+            {
+                try
+                {
+                    return await SendAsync(method, path, data, compress);
+                }
+                catch (Exception ex)
+                {
+                    error = ex;
+                }
+            }
+
+            throw error;
+        }
+
+        public async Task<byte[]> GetAsync(string path)
+        {
+            return await SendWithRetriesAsync(HttpMethod.Get, path, null);
         }
 
         public byte[] Get(string path)
         {
-            var error = new Exception("Internal error");
-
-            // NOTE: <= is intentional, 0 is send, 1,2,3 is retry
-            for (var i = 0; i <= _retries; ++i)
-            {
-                try
-                {
-                    return Send(HttpMethod.Get, path, null, false);
-                }
-                catch (Exception ex)
-                {
-                    error = ex;
-                }
+            return Task.Run(() => GetAsync(path)).Result;
         }
 
-            throw error;
+        public async Task<byte[]> PostAsync(string path, byte[] data, bool compress = true)
+        {
+            return await SendWithRetriesAsync(HttpMethod.Post, path, data, compress);
         }
 
-        public byte[] Post(string path, byte[] data, bool compressed = true)
+        public byte[] Post(string path, byte[] data, bool compress = true)
         {
-            var error = new Exception("Internal error");
-
-            // NOTE: <= is intentional, 0 is send, 1,2,3 is retry
-            for (var i = 0; i <= _retries; ++i)
-            {
-                try
-                {
-                    return Send(HttpMethod.Post, path, data, compressed);
-                }
-                catch (Exception ex)
-                {
-                    error = ex;
-                }
+            return Task.Run(() => PostAsync(path, data, compress)).Result;
         }
 
-            throw error;
+        // NOTE: returns status code as bytes
+        public async Task<byte[]> PutAsync(string path, byte[] data, bool compress = true)
+        {
+            return await SendWithRetriesAsync(HttpMethod.Post, path, data, compress);
         }
 
-        public void Put(string path, byte[] data, bool compressed = true)
+        // NOTE: returns status code as bytes
+        public byte[] Put(string path, byte[] data, bool compress = true)
         {
-            var error = new Exception("Internal error");
-
-            // NOTE: <= is intentional, 0 is send, 1,2,3 is retry
-            for (var i = 0; i <= _retries; ++i)
-            {
-                try
-                {
-                    Send(HttpMethod.Put, path, data, compressed);
-                    return;
-                }
-                catch (Exception ex)
-                {
-                    error = ex;
-                }
-            }
-
-            throw error;
+            return Task.Run(() => PutAsync(path, data, compress)).Result;
         }
 
         public void Dispose()

project/Aki.Common/Http/RequestHandler.cs

@@ -43,65 +43,91 @@ namespace Aki.Common.Http
             HttpClient = new Client(Host, SessionId);
         }
 
-        private static void ValidateData(byte[] data)
+        private static void ValidateData(string path, byte[] data)
         {
             if (data == null)
             {
-                _logger.LogError($"Request failed, body is null");
+                _logger.LogError($"[REQUEST FAILED] {path}");
             }
 
-            _logger.LogInfo($"Request was successful");
+            _logger.LogInfo($"[REQUEST SUCCESSFUL] {path}");
         }
 
-        private static void ValidateJson(string json)
+        private static void ValidateJson(string path, string json)
         {
             if (string.IsNullOrWhiteSpace(json))
             {
-                _logger.LogError($"Request failed, body is null");
+                _logger.LogError($"[REQUEST FAILED] {path}");
             }
 
-            _logger.LogInfo($"Request was successful");
+            _logger.LogInfo($"[REQUEST SUCCESSFUL] {path}");
+        }
+
+        public static async Task<byte[]> GetDataAsync(string path)
+        {
+            _logger.LogInfo($"[REQUEST]: {path}");
+            
+            var data = await HttpClient.GetAsync(path);
+
+            ValidateData(path, data);
+            return data;
         }
 
         public static byte[] GetData(string path)
         {
-            _logger.LogInfo($"Request GET data: {SessionId}:{path}");
+            return Task.Run(() => GetData(path)).Result;
+        }
 
-            var data = HttpClient.Get(path);
+        public static async Task<string> GetJsonAsync(string path)
+        {
+            _logger.LogInfo($"[REQUEST]: {path}");
             
-            ValidateData(data);
-            return data;
+            var payload = await HttpClient.GetAsync(path);
+            var body = Encoding.UTF8.GetString(payload);
+
+            ValidateJson(path, body);
+            return body;
         }
 
         public static string GetJson(string path)
         {
-            _logger.LogInfo($"Request GET json: {SessionId}:{path}");
-            
-            var payload = HttpClient.Get(path);
-            var body = Encoding.UTF8.GetString(payload);
-
-            ValidateJson(body);
-            return body;
+            return Task.Run(() => GetJsonAsync(path)).Result;
         }
 
-        public static string PostJson(string path, string json)
+        public static string PostJsonAsync(string path, string json)
         {
-            _logger.LogInfo($"Request POST json: {SessionId}:{path}");
+            _logger.LogInfo($"[REQUEST]: {path}");
             
             var payload = Encoding.UTF8.GetBytes(json);
             var data = HttpClient.Post(path, payload);
             var body = Encoding.UTF8.GetString(data);
 
-            ValidateJson(body);
+            ValidateJson(path, body);
             return body;
         }
 
-        public static void PutJson(string path, string json)
+        public static string PostJson(string path, string json)
         {
-            _logger.LogInfo($"Request PUT json: {SessionId}:{path}");
+            return Task.Run(() => PostJsonAsync(path, json)).Result;
+        }
+
+        // NOTE: returns status code
+        public static async Task<string> PutJsonAsync(string path, string json)
+        {
+            _logger.LogInfo($"[REQUEST]: {path}");
 
             var payload = Encoding.UTF8.GetBytes(json);
-            HttpClient.Put(path, payload);
+            var data = await HttpClient.PutAsync(path, payload);
+            var body = Encoding.UTF8.GetString(data);
+
+            ValidateJson(path, body);
+            return body;
+        }
+
+        // NOTE: returns status code
+        public static string PutJson(string path, string json)
+        {
+            return Task.Run(() => PutJsonAsync(path, json)).Result;
         }
 
 #region DEPRECATED, REMOVE IN 3.8.1
@@ -120,7 +146,7 @@ namespace Aki.Common.Http
             var request = new Request();
             var data = request.Send(url, "GET", null, headers: headers);
 
-            ValidateData(data);
+            ValidateData(url, data);
             return data;
 
         }
@@ -141,7 +167,7 @@ namespace Aki.Common.Http
             var data = request.Send(url, "GET", headers: headers);
             var body = Encoding.UTF8.GetString(data);
 
-            ValidateJson(body);
+            ValidateJson(url, body);
             return body;
 
         }
@@ -164,7 +190,7 @@ namespace Aki.Common.Http
             var data = request.Send(url, "POST", payload, true, mime, headers);
             var body = Encoding.UTF8.GetString(data);
 
-            ValidateJson(body);
+            ValidateJson(url, body);
             return body;
 
         }

project/Aki.Common/Utils/VFS.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Threading.Tasks;
 
 namespace Aki.Common.Utils
 {
@@ -97,6 +98,22 @@ namespace Aki.Common.Utils
             return File.ReadAllBytes(filepath);
         }
 
+        /// <summary>
+        /// Get file content as bytes.
+        /// </summary>
+        public static async Task<byte[]> ReadFileAsync(string filepath)
+        {
+            byte[] result;
+
+            using (var fs = File.Open(filepath, FileMode.Open))
+            {
+                result = new byte[fs.Length];
+                await fs.ReadAsync(result, 0, (int)fs.Length);
+            }
+
+            return result;
+        }
+
         /// <summary>
         /// Get file content as string.
         /// </summary>
@@ -105,6 +122,20 @@ namespace Aki.Common.Utils
             return File.ReadAllText(filepath);
         }
 
+        /// <summary>
+        /// Get file content as string.
+        /// </summary>
+        public static async Task<string> ReadTextFileAsync(string filepath)
+        {
+            using (var fs = File.Open(filepath, FileMode.Open))
+            {
+                using (var sr = new StreamReader(fs))
+                {
+                    return await sr.ReadToEndAsync();
+                }
+            }
+        }
+
         /// <summary>
         /// Write data to file.
         /// </summary>
@@ -118,6 +149,22 @@ namespace Aki.Common.Utils
             File.WriteAllBytes(filepath, data);
         }
 
+        /// <summary>
+        /// Write data to file.
+        /// </summary>
+        public static async Task WriteFileAsync(string filepath, byte[] data)
+        {
+            if (!Exists(filepath))
+            {
+                CreateDirectory(filepath.GetDirectory());
+            }
+
+            using (FileStream stream = File.Open(filepath, FileMode.OpenOrCreate, FileAccess.Write))
+            {
+                await stream.WriteAsync(data, 0, data.Length);
+            }
+        }
+
         /// <summary>
         /// Write string to file.
         /// </summary>

project/Aki.Custom/AkiCustomPlugin.cs

@@ -21,7 +21,6 @@ namespace Aki.Custom
             try
             {
                 // Bundle patches should always load first
-                BundleManager.GetBundles();
                 new EasyAssetsPatch().Enable();
                 new EasyBundlePatch().Enable();
 

project/Aki.Custom/Patches/EasyAssetsPatch.cs

@@ -1,21 +1,19 @@
-using Aki.Reflection.Patching;
-using Diz.Jobs;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Threading.Tasks;
 using Diz.Resources;
 using JetBrains.Annotations;
 using Newtonsoft.Json;
 using UnityEngine;
 using UnityEngine.Build.Pipeline;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Reflection;
-using System.Threading.Tasks;
 using Aki.Common.Utils;
 using Aki.Custom.Models;
 using Aki.Custom.Utils;
-using DependencyGraph = DependencyGraph<IEasyBundle>;
+using Aki.Reflection.Patching;
 using Aki.Reflection.Utils;
+using DependencyGraph = DependencyGraph<IEasyBundle>;
 
 namespace Aki.Custom.Patches
 {
@@ -38,36 +36,35 @@ namespace Aki.Custom.Patches
 
         protected override MethodBase GetTargetMethod()
         {
-            return typeof(EasyAssets).GetMethods(PatchConstants.PublicDeclaredFlags).SingleCustom(IsTargetMethod);
-        }
-
-        private static bool IsTargetMethod(MethodInfo mi)
-        {
-            var parameters = mi.GetParameters();
-            return (parameters.Length == 6
-                    && parameters[0].Name == "bundleLock"
-                    && parameters[1].Name == "defaultKey"
-                    && parameters[4].Name == "shouldExclude");
+            return typeof(EasyAssets).GetMethod(nameof(EasyAssets.Create));
         }
 
         [PatchPrefix]
-        private static bool PatchPrefix(ref Task __result, EasyAssets __instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath,
+        private static bool PatchPrefix(ref Task<EasyAssets> __result, GameObject gameObject, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath,
             string platformName, [CanBeNull] Func<string, bool> shouldExclude, [CanBeNull] Func<string, Task> bundleCheck)
         {
-            __result = Init(__instance, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck);
+            var easyAsset = gameObject.AddComponent<EasyAssets>();
+            __result = Init(easyAsset, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck);
+
             return false;
         }
 
-        private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func<string, bool> shouldExclude, Func<string, Task> bundleCheck)
+        private static async Task<EasyAssets> Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func<string, bool> shouldExclude, Func<string, Task> bundleCheck)
         {
             // platform manifest
-            var path = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/";
-            var filepath = path + platformName;
+            var eftBundlesPath = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/";
+            var filepath = eftBundlesPath + platformName;
             var jsonfile = filepath + ".json";
-            var manifest = File.Exists(jsonfile)
+            var manifest = VFS.Exists(jsonfile)
                 ? await GetManifestJson(jsonfile)
                 : await GetManifestBundle(filepath);
 
+            // lazy-initialize aki bundles
+            if (BundleManager.Bundles.Keys.Count == 0)
+            {
+                await BundleManager.DownloadManifest();
+            }
+
             // create bundles array from obfuscated type
             var bundleNames = manifest.GetAllAssetBundles()
                 .Union(BundleManager.Bundles.Keys)
@@ -79,27 +76,43 @@ namespace Aki.Custom.Patches
                 bundleLock = new BundleLock(int.MaxValue);
             }
 
-            // create bundle of obfuscated type
             var bundles = (IEasyBundle[])Array.CreateInstance(EasyBundleHelper.Type, bundleNames.Length);
 
             for (var i = 0; i < bundleNames.Length; i++)
             {
+                var key = bundleNames[i];
+                var path = eftBundlesPath;
+
+                // acquire external bundle
+                if (BundleManager.Bundles.TryGetValue(key, out var bundleInfo))
+                {
+                    // we need base path without file extension
+                    path = BundleManager.GetBundlePath(bundleInfo);
+
+                    // only download when connected externally
+                    if (await BundleManager.ShouldReaquire(bundleInfo))
+                    {
+                        await BundleManager.DownloadBundle(bundleInfo);
+                    }
+                }
+
+                // create bundle of obfuscated type
                 bundles[i] = (IEasyBundle)Activator.CreateInstance(EasyBundleHelper.Type, new object[]
                 {
-                    bundleNames[i],
+                    key,
                     path,
                     manifest,
                     bundleLock,
                     bundleCheck
                 });
-
-                await JobScheduler.Yield(EJobPriority.Immediate);
             }
 
             // create dependency graph
             instance.Manifest = manifest;
             _bundlesField.SetValue(instance, bundles);
             instance.System = new DependencyGraph(bundles, defaultKey, shouldExclude);
+
+            return instance;
         }
 
         // NOTE: used by:
@@ -122,7 +135,7 @@ namespace Aki.Custom.Patches
 
         private static async Task<CompatibilityAssetBundleManifest> GetManifestJson(string filepath)
         {
-            var text = VFS.ReadTextFile(filepath);
+            var text = await VFS.ReadTextFileAsync(filepath);
 
             /* we cannot parse directly as <string, BundleDetails>, because...
                     [Error  : Unity Log] JsonSerializationException: Expected string when reading UnityEngine.Hash128 type, got 'StartObject' <>. Path '['assets/content/weapons/animations/simple_animations.bundle'].Hash', line 1, position 176.

project/Aki.Custom/Patches/EasyBundlePatch.cs

@@ -1,12 +1,12 @@
 using System;
-using Aki.Reflection.Patching;
-using Diz.DependencyManager;
-using UnityEngine.Build.Pipeline;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using Diz.DependencyManager;
+using UnityEngine.Build.Pipeline;
 using Aki.Custom.Models;
 using Aki.Custom.Utils;
+using Aki.Reflection.Patching;
 
 namespace Aki.Custom.Patches
 {
@@ -38,7 +38,7 @@ namespace Aki.Custom.Patches
                     : bundle.Dependencies;
 
                 // set path to either cache (HTTP) or mod (local)
-                filepath = BundleManager.GetBundlePath(bundle);
+                filepath = BundleManager.GetBundleFilePath(bundle);
             }
 
             _ = new EasyBundleHelper(__instance)

project/Aki.Custom/Utils/BundleManager.cs

@@ -10,85 +10,66 @@ namespace Aki.Custom.Utils
 {
     public static class BundleManager
     {
-        private static ManualLogSource _logger;
+        private const string CachePath = "user/cache/bundles/";
+        private static readonly ManualLogSource _logger;
         public static readonly ConcurrentDictionary<string, BundleItem> Bundles;
-        public static string CachePath;
 
         static BundleManager()
         {
             _logger = Logger.CreateLogSource(nameof(BundleManager));
             Bundles = new ConcurrentDictionary<string, BundleItem>();
-            CachePath = "user/cache/bundles/";
         }
 
         public static string GetBundlePath(BundleItem bundle)
         {
             return RequestHandler.IsLocal
-                ? $"{bundle.ModPath}/bundles/{bundle.FileName}"
-                : CachePath + bundle.FileName;
+                ? $"{bundle.ModPath}/bundles/"
+                : CachePath;
         }
 
-        public static void GetBundles()
+        public static string GetBundleFilePath(BundleItem bundle)
+        {
+            return GetBundlePath(bundle) + bundle.FileName;
+        }
+
+        public static async Task DownloadManifest()
         {
             // get bundles
-            var json = RequestHandler.GetJson("/singleplayer/bundles");
+            var json = await RequestHandler.GetJsonAsync("/singleplayer/bundles");
             var bundles = JsonConvert.DeserializeObject<BundleItem[]>(json);
 
-            // register bundles
-            var toDownload = new ConcurrentBag<BundleItem>();
-
-            Parallel.ForEach(bundles, (bundle) =>
+            foreach (var bundle in bundles)
             {
                 Bundles.TryAdd(bundle.FileName, bundle);
-
-                if (ShouldReaquire(bundle))
-                {
-                    // mark for download
-                    toDownload.Add(bundle);
-                }
-            });
-
-            if (RequestHandler.IsLocal)
-            {
-                // loading from local mods
-                _logger.LogInfo("CACHE: Loading all bundles from mods on disk.");
-                return;
-            }
-            else
-            {
-                // download bundles
-                // NOTE: assumes bundle keys to be unique
-                Parallel.ForEach(toDownload, (bundle) =>
-                {
-                    // download bundle
-                    var filepath = GetBundlePath(bundle);
-                    var data = RequestHandler.GetData($"/files/bundle/{bundle.FileName}");
-                    VFS.WriteFile(filepath, data);
-                });
             }
         }
 
-        private static bool ShouldReaquire(BundleItem bundle)
+        public static async Task DownloadBundle(BundleItem bundle)
         {
-            if (RequestHandler.IsLocal)
-            {
-                // only handle remote bundles
-                return false;
+            var filepath = GetBundleFilePath(bundle);
+            var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}");
+            await VFS.WriteFileAsync(filepath, data);
         }
 
+        public static async Task<bool> ShouldReaquire(BundleItem bundle)
+        {
             // read cache
-            var filepath = CachePath + bundle.FileName;
+            var filepath = GetBundleFilePath(bundle);
 
             if (VFS.Exists(filepath))
             {
                 // calculate hash
-                var data = VFS.ReadFile(filepath);
+                var data = await VFS.ReadFileAsync(filepath);
                 var crc = Crc32.Compute(data);
 
                 if (crc == bundle.Crc)
                 {
                     // file is up-to-date
-                    _logger.LogInfo($"CACHE: Loading locally {bundle.FileName}");
+                    var location = RequestHandler.IsLocal
+                        ? "MOD"
+                        : "CACHE";
+
+                    _logger.LogInfo($"{location}: Loading locally {bundle.FileName}");
                     return false;
                 }
                 else