From 4b401e7449d128d6cc4930515e7cb20fd1e4c6f7 Mon Sep 17 00:00:00 2001 From: Merijn Hendriks Date: Mon, 6 May 2024 10:28:51 +0000 Subject: [PATCH 1/5] async-bundles (!117) This patch contains the following: - Initial async VFS code (for reading / writing files) - Simplified Http Client code - Added async support to Http Client, RequestHandler - Improved RequestHandler logging - Deferred bundle loading to EasyAssetPatch - Make GetManifestJson run async This comes with a number of benefits: - When downloading bundles, it will mention which files succeeded or failed to download - Bundle loading happens in the initial screen, not the white screen - Fixed the issue where bundle loading could break bepinex loading (too long load time) - Modders can now make async http request and read/write files async I removed logging of sessionid inside the RequestHandler for each request, sessionid is already visible from bepinex log startup parameters. At last, sorry for the amount of commits it took. I initially wanted to target the 3.9.0 branch, but decided to use 3.8.1 instead as async request can really help out some mods. Reviewed-on: https://dev.sp-tarkov.com/SPT-AKI/Modules/pulls/117 Co-authored-by: Merijn Hendriks Co-committed-by: Merijn Hendriks --- project/Aki.Common/Http/Client.cs | 140 +++++++++--------- project/Aki.Common/Http/RequestHandler.cs | 80 ++++++---- project/Aki.Common/Utils/VFS.cs | 47 ++++++ project/Aki.Custom/AkiCustomPlugin.cs | 1 - project/Aki.Custom/Patches/EasyAssetsPatch.cs | 8 +- project/Aki.Custom/Utils/BundleManager.cs | 22 +-- 6 files changed, 185 insertions(+), 113 deletions(-) diff --git a/project/Aki.Common/Http/Client.cs b/project/Aki.Common/Http/Client.cs index 993f1c1..4468c1f 100644 --- a/project/Aki.Common/Http/Client.cs +++ b/project/Aki.Common/Http/Client.cs @@ -1,14 +1,14 @@ using System; using System.IO; using System.Net.Http; -using Aki.Common.Http; +using System.Text; +using System.Threading.Tasks; using Aki.Common.Utils; namespace Aki.Common.Http { - // NOTE: you do not want to dispose this, keep a reference for the lifetime - // of the application. - // NOTE: cannot be made async due to Unity's limitations. + // NOTE: Don't dispose this, keep a reference for the lifetime of the + // application. public class Client : IDisposable { protected readonly HttpClient _httpv; @@ -22,9 +22,9 @@ namespace Aki.Common.Http _accountId = accountId; _retries = retries; - var handler = new HttpClientHandler() + var handler = new HttpClientHandler { - // force setting cookies in header instead of CookieContainer + // set cookies in header instead UseCookies = false }; @@ -43,7 +43,7 @@ namespace Aki.Common.Http }; } - protected byte[] Send(HttpMethod method, string path, byte[] data, bool compress = true) + protected async Task SendAsync(HttpMethod method, string path, byte[] data, bool zipped = true) { HttpResponseMessage response = null; @@ -51,17 +51,17 @@ namespace Aki.Common.Http { if (data != null) { - // if there is data, convert to payload - byte[] payload = (compress) - ? Zlib.Compress(data, ZlibCompression.Maximum) - : data; - // add payload to request - request.Content = new ByteArrayContent(payload); + if (zipped) + { + data = Zlib.Compress(data, ZlibCompression.Maximum); + } + + request.Content = new ByteArrayContent(data); } // send request - response = _httpv.SendAsync(request).Result; + response = await _httpv.SendAsync(request); } if (!response.IsSuccessStatusCode) @@ -72,85 +72,79 @@ namespace Aki.Common.Http using (var ms = new MemoryStream()) { - using (var stream = response.Content.ReadAsStreamAsync().Result) + using (var stream = await response.Content.ReadAsStreamAsync()) { // grap response payload - stream.CopyTo(ms); - var bytes = ms.ToArray(); + await stream.CopyToAsync(ms); + var body = ms.ToArray(); - if (bytes != null) + if (Zlib.IsCompressed(body)) { - // payload contains data - return Zlib.IsCompressed(bytes) - ? Zlib.Decompress(bytes) - : bytes; + body = Zlib.Decompress(body); } + + if (body == null) + { + // payload doesn't contains data + var code = response.StatusCode.ToString(); + body = Encoding.UTF8.GetBytes(code); + } + + return body; + } + } + } + + protected async Task SendWithRetriesAsync(HttpMethod method, string path, byte[] data, bool compress = true) + { + var error = new Exception("Internal error"); + + // NOTE: <= is intentional. 0 is send, 1/2/3 is retry + for (var i = 0; i <= _retries; ++i) + { + try + { + return await SendAsync(method, path, data, compress); + } + catch (Exception ex) + { + error = ex; } } - // response returned no data - return null; + throw error; + } + + public async Task GetAsync(string path) + { + return await SendWithRetriesAsync(HttpMethod.Get, path, null); } public byte[] Get(string path) { - var error = new Exception("Internal error"); - - // NOTE: <= is intentional, 0 is send, 1,2,3 is retry - for (var i = 0; i <= _retries; ++i) - { - try - { - return Send(HttpMethod.Get, path, null, false); - } - catch (Exception ex) - { - error = ex; - } - } - - throw error; + return Task.Run(() => GetAsync(path)).Result; } - public byte[] Post(string path, byte[] data, bool compressed = true) + public async Task PostAsync(string path, byte[] data, bool compress = true) { - var error = new Exception("Internal error"); - - // NOTE: <= is intentional, 0 is send, 1,2,3 is retry - for (var i = 0; i <= _retries; ++i) - { - try - { - return Send(HttpMethod.Post, path, data, compressed); - } - catch (Exception ex) - { - error = ex; - } - } - - throw error; + return await SendWithRetriesAsync(HttpMethod.Post, path, data, compress); } - public void Put(string path, byte[] data, bool compressed = true) + public byte[] Post(string path, byte[] data, bool compress = true) { - var error = new Exception("Internal error"); + return Task.Run(() => PostAsync(path, data, compress)).Result; + } - // NOTE: <= is intentional, 0 is send, 1,2,3 is retry - for (var i = 0; i <= _retries; ++i) - { - try - { - Send(HttpMethod.Put, path, data, compressed); - return; - } - catch (Exception ex) - { - error = ex; - } - } + // NOTE: returns status code as bytes + public async Task PutAsync(string path, byte[] data, bool compress = true) + { + return await SendWithRetriesAsync(HttpMethod.Post, path, data, compress); + } - throw error; + // NOTE: returns status code as bytes + public byte[] Put(string path, byte[] data, bool compress = true) + { + return Task.Run(() => PutAsync(path, data, compress)).Result; } public void Dispose() diff --git a/project/Aki.Common/Http/RequestHandler.cs b/project/Aki.Common/Http/RequestHandler.cs index 22f5d65..498ecee 100644 --- a/project/Aki.Common/Http/RequestHandler.cs +++ b/project/Aki.Common/Http/RequestHandler.cs @@ -43,65 +43,91 @@ namespace Aki.Common.Http HttpClient = new Client(Host, SessionId); } - private static void ValidateData(byte[] data) + private static void ValidateData(string path, byte[] data) { if (data == null) { - _logger.LogError($"Request failed, body is null"); + _logger.LogError($"[REQUEST FAILED] {path}"); } - _logger.LogInfo($"Request was successful"); + _logger.LogInfo($"[REQUEST SUCCESSFUL] {path}"); } - private static void ValidateJson(string json) + private static void ValidateJson(string path, string json) { if (string.IsNullOrWhiteSpace(json)) { - _logger.LogError($"Request failed, body is null"); + _logger.LogError($"[REQUEST FAILED] {path}"); } - _logger.LogInfo($"Request was successful"); + _logger.LogInfo($"[REQUEST SUCCESSFUL] {path}"); + } + + public static async Task GetDataAsync(string path) + { + _logger.LogInfo($"[REQUEST]: {path}"); + + var data = await HttpClient.GetAsync(path); + + ValidateData(path, data); + return data; } public static byte[] GetData(string path) { - _logger.LogInfo($"Request GET data: {SessionId}:{path}"); - - var data = HttpClient.Get(path); + return Task.Run(() => GetData(path)).Result; + } - ValidateData(data); - return data; + public static async Task GetJsonAsync(string path) + { + _logger.LogInfo($"[REQUEST]: {path}"); + + var payload = await HttpClient.GetAsync(path); + var body = Encoding.UTF8.GetString(payload); + + ValidateJson(path, body); + return body; } public static string GetJson(string path) { - _logger.LogInfo($"Request GET json: {SessionId}:{path}"); - - var payload = HttpClient.Get(path); - var body = Encoding.UTF8.GetString(payload); - - ValidateJson(body); - return body; + return Task.Run(() => GetJsonAsync(path)).Result; } - public static string PostJson(string path, string json) + public static string PostJsonAsync(string path, string json) { - _logger.LogInfo($"Request POST json: {SessionId}:{path}"); + _logger.LogInfo($"[REQUEST]: {path}"); var payload = Encoding.UTF8.GetBytes(json); var data = HttpClient.Post(path, payload); var body = Encoding.UTF8.GetString(data); - ValidateJson(body); + ValidateJson(path, body); return body; } - public static void PutJson(string path, string json) + public static string PostJson(string path, string json) { - _logger.LogInfo($"Request PUT json: {SessionId}:{path}"); + return Task.Run(() => PostJsonAsync(path, json)).Result; + } + + // NOTE: returns status code + public static async Task PutJsonAsync(string path, string json) + { + _logger.LogInfo($"[REQUEST]: {path}"); var payload = Encoding.UTF8.GetBytes(json); - HttpClient.Put(path, payload); + var data = await HttpClient.PutAsync(path, payload); + var body = Encoding.UTF8.GetString(data); + + ValidateJson(path, body); + return body; + } + + // NOTE: returns status code + public static string PutJson(string path, string json) + { + return Task.Run(() => PutJsonAsync(path, json)).Result; } #region DEPRECATED, REMOVE IN 3.8.1 @@ -120,7 +146,7 @@ namespace Aki.Common.Http var request = new Request(); var data = request.Send(url, "GET", null, headers: headers); - ValidateData(data); + ValidateData(url, data); return data; } @@ -141,7 +167,7 @@ namespace Aki.Common.Http var data = request.Send(url, "GET", headers: headers); var body = Encoding.UTF8.GetString(data); - ValidateJson(body); + ValidateJson(url, body); return body; } @@ -164,7 +190,7 @@ namespace Aki.Common.Http var data = request.Send(url, "POST", payload, true, mime, headers); var body = Encoding.UTF8.GetString(data); - ValidateJson(body); + ValidateJson(url, body); return body; } diff --git a/project/Aki.Common/Utils/VFS.cs b/project/Aki.Common/Utils/VFS.cs index b241c00..74f814b 100644 --- a/project/Aki.Common/Utils/VFS.cs +++ b/project/Aki.Common/Utils/VFS.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Threading.Tasks; namespace Aki.Common.Utils { @@ -97,6 +98,22 @@ namespace Aki.Common.Utils return File.ReadAllBytes(filepath); } + /// + /// Get file content as bytes. + /// + public static async Task ReadFileAsync(string filepath) + { + byte[] result; + + using (var fs = File.Open(filepath, FileMode.Open)) + { + result = new byte[fs.Length]; + await fs.ReadAsync(result, 0, (int)fs.Length); + } + + return result; + } + /// /// Get file content as string. /// @@ -105,6 +122,20 @@ namespace Aki.Common.Utils return File.ReadAllText(filepath); } + /// + /// Get file content as string. + /// + public static async Task ReadTextFileAsync(string filepath) + { + using (var fs = File.Open(filepath, FileMode.Open)) + { + using (var sr = new StreamReader(fs)) + { + return await sr.ReadToEndAsync(); + } + } + } + /// /// Write data to file. /// @@ -118,6 +149,22 @@ namespace Aki.Common.Utils File.WriteAllBytes(filepath, data); } + /// + /// Write data to file. + /// + public static async Task WriteFileAsync(string filepath, byte[] data) + { + if (!Exists(filepath)) + { + CreateDirectory(filepath.GetDirectory()); + } + + using (FileStream stream = File.Open(filepath, FileMode.OpenOrCreate, FileAccess.Write)) + { + await stream.WriteAsync(data, 0, data.Length); + } + } + /// /// Write string to file. /// diff --git a/project/Aki.Custom/AkiCustomPlugin.cs b/project/Aki.Custom/AkiCustomPlugin.cs index e3d39f8..0c4e4ae 100644 --- a/project/Aki.Custom/AkiCustomPlugin.cs +++ b/project/Aki.Custom/AkiCustomPlugin.cs @@ -21,7 +21,6 @@ namespace Aki.Custom try { // Bundle patches should always load first - BundleManager.GetBundles(); new EasyAssetsPatch().Enable(); new EasyBundlePatch().Enable(); diff --git a/project/Aki.Custom/Patches/EasyAssetsPatch.cs b/project/Aki.Custom/Patches/EasyAssetsPatch.cs index 7f4fa16..3ebeb97 100644 --- a/project/Aki.Custom/Patches/EasyAssetsPatch.cs +++ b/project/Aki.Custom/Patches/EasyAssetsPatch.cs @@ -68,6 +68,12 @@ namespace Aki.Custom.Patches ? await GetManifestJson(jsonfile) : await GetManifestBundle(filepath); + // lazy-initialize aki bundles + if (BundleManager.Bundles.Keys.Count == 0) + { + await BundleManager.GetBundles(); + } + // create bundles array from obfuscated type var bundleNames = manifest.GetAllAssetBundles() .Union(BundleManager.Bundles.Keys) @@ -122,7 +128,7 @@ namespace Aki.Custom.Patches private static async Task GetManifestJson(string filepath) { - var text = VFS.ReadTextFile(filepath); + var text = await VFS.ReadTextFileAsync(filepath); /* we cannot parse directly as , because... [Error : Unity Log] JsonSerializationException: Expected string when reading UnityEngine.Hash128 type, got 'StartObject' <>. Path '['assets/content/weapons/animations/simple_animations.bundle'].Hash', line 1, position 176. diff --git a/project/Aki.Custom/Utils/BundleManager.cs b/project/Aki.Custom/Utils/BundleManager.cs index caf11ac..c2e13a9 100644 --- a/project/Aki.Custom/Utils/BundleManager.cs +++ b/project/Aki.Custom/Utils/BundleManager.cs @@ -10,7 +10,7 @@ namespace Aki.Custom.Utils { public static class BundleManager { - private static ManualLogSource _logger; + private static readonly ManualLogSource _logger; public static readonly ConcurrentDictionary Bundles; public static string CachePath; @@ -28,7 +28,7 @@ namespace Aki.Custom.Utils : CachePath + bundle.FileName; } - public static void GetBundles() + public static async Task GetBundles() { // get bundles var json = RequestHandler.GetJson("/singleplayer/bundles"); @@ -37,16 +37,16 @@ namespace Aki.Custom.Utils // register bundles var toDownload = new ConcurrentBag(); - Parallel.ForEach(bundles, (bundle) => + foreach (var bundle in bundles) { Bundles.TryAdd(bundle.FileName, bundle); - if (ShouldReaquire(bundle)) + if (await ShouldReaquire(bundle)) { // mark for download toDownload.Add(bundle); } - }); + } if (RequestHandler.IsLocal) { @@ -58,17 +58,17 @@ namespace Aki.Custom.Utils { // download bundles // NOTE: assumes bundle keys to be unique - Parallel.ForEach(toDownload, (bundle) => + foreach (var bundle in toDownload) { // download bundle var filepath = GetBundlePath(bundle); - var data = RequestHandler.GetData($"/files/bundle/{bundle.FileName}"); - VFS.WriteFile(filepath, data); - }); + var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); + await VFS.WriteFileAsync(filepath, data); + } } } - private static bool ShouldReaquire(BundleItem bundle) + private static async Task ShouldReaquire(BundleItem bundle) { if (RequestHandler.IsLocal) { @@ -82,7 +82,7 @@ namespace Aki.Custom.Utils if (VFS.Exists(filepath)) { // calculate hash - var data = VFS.ReadFile(filepath); + var data = await VFS.ReadFileAsync(filepath); var crc = Crc32.Compute(data); if (crc == bundle.Crc) From 9e65e68c81f47b05b29244f08d3cb2b9edc8616d Mon Sep 17 00:00:00 2001 From: Merijn Hendriks Date: Mon, 6 May 2024 19:44:28 +0000 Subject: [PATCH 2/5] Improve async bundles (!123) Synchronizes the changes with NoHurry.AsyncBundleLoader. A large chunk of this coded has been tested at scale in Fika with very good results. - Targets `EasyAssets.Create` instead of using a complex lookup pattern - Delays downloading of bundles to initial loading screen - Bundles are downloaded using intended sequence - Improved logging of bundle loading Reviewed-on: https://dev.sp-tarkov.com/SPT-AKI/Modules/pulls/123 Co-authored-by: Merijn Hendriks Co-committed-by: Merijn Hendriks --- project/Aki.Custom/Patches/EasyAssetsPatch.cs | 67 ++++++++++--------- project/Aki.Custom/Patches/EasyBundlePatch.cs | 8 +-- project/Aki.Custom/Utils/BundleManager.cs | 63 ++++++----------- 3 files changed, 63 insertions(+), 75 deletions(-) diff --git a/project/Aki.Custom/Patches/EasyAssetsPatch.cs b/project/Aki.Custom/Patches/EasyAssetsPatch.cs index 3ebeb97..c62465c 100644 --- a/project/Aki.Custom/Patches/EasyAssetsPatch.cs +++ b/project/Aki.Custom/Patches/EasyAssetsPatch.cs @@ -1,21 +1,19 @@ -using Aki.Reflection.Patching; -using Diz.Jobs; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; using Diz.Resources; using JetBrains.Annotations; using Newtonsoft.Json; using UnityEngine; using UnityEngine.Build.Pipeline; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; using Aki.Common.Utils; using Aki.Custom.Models; using Aki.Custom.Utils; -using DependencyGraph = DependencyGraph; +using Aki.Reflection.Patching; using Aki.Reflection.Utils; +using DependencyGraph = DependencyGraph; namespace Aki.Custom.Patches { @@ -38,40 +36,33 @@ namespace Aki.Custom.Patches protected override MethodBase GetTargetMethod() { - return typeof(EasyAssets).GetMethods(PatchConstants.PublicDeclaredFlags).SingleCustom(IsTargetMethod); - } - - private static bool IsTargetMethod(MethodInfo mi) - { - var parameters = mi.GetParameters(); - return (parameters.Length == 6 - && parameters[0].Name == "bundleLock" - && parameters[1].Name == "defaultKey" - && parameters[4].Name == "shouldExclude"); + return typeof(EasyAssets).GetMethod(nameof(EasyAssets.Create)); } [PatchPrefix] - private static bool PatchPrefix(ref Task __result, EasyAssets __instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, + private static bool PatchPrefix(ref Task __result, GameObject gameObject, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, [CanBeNull] Func bundleCheck) { - __result = Init(__instance, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); + var easyAsset = gameObject.AddComponent(); + __result = Init(easyAsset, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); + return false; } - private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) + private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) { // platform manifest - var path = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; - var filepath = path + platformName; + var eftBundlesPath = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; + var filepath = eftBundlesPath + platformName; var jsonfile = filepath + ".json"; - var manifest = File.Exists(jsonfile) + var manifest = VFS.Exists(jsonfile) ? await GetManifestJson(jsonfile) : await GetManifestBundle(filepath); // lazy-initialize aki bundles if (BundleManager.Bundles.Keys.Count == 0) { - await BundleManager.GetBundles(); + await BundleManager.DownloadManifest(); } // create bundles array from obfuscated type @@ -85,27 +76,43 @@ namespace Aki.Custom.Patches bundleLock = new BundleLock(int.MaxValue); } - // create bundle of obfuscated type var bundles = (IEasyBundle[])Array.CreateInstance(EasyBundleHelper.Type, bundleNames.Length); for (var i = 0; i < bundleNames.Length; i++) { + var key = bundleNames[i]; + var path = eftBundlesPath; + + // acquire external bundle + if (BundleManager.Bundles.TryGetValue(key, out var bundleInfo)) + { + // we need base path without file extension + path = BundleManager.GetBundlePath(bundleInfo); + + // only download when connected externally + if (await BundleManager.ShouldReaquire(bundleInfo)) + { + await BundleManager.DownloadBundle(bundleInfo); + } + } + + // create bundle of obfuscated type bundles[i] = (IEasyBundle)Activator.CreateInstance(EasyBundleHelper.Type, new object[] { - bundleNames[i], + key, path, manifest, bundleLock, bundleCheck }); - - await JobScheduler.Yield(EJobPriority.Immediate); } // create dependency graph instance.Manifest = manifest; _bundlesField.SetValue(instance, bundles); instance.System = new DependencyGraph(bundles, defaultKey, shouldExclude); + + return instance; } // NOTE: used by: diff --git a/project/Aki.Custom/Patches/EasyBundlePatch.cs b/project/Aki.Custom/Patches/EasyBundlePatch.cs index f5e727c..b95593d 100644 --- a/project/Aki.Custom/Patches/EasyBundlePatch.cs +++ b/project/Aki.Custom/Patches/EasyBundlePatch.cs @@ -1,12 +1,12 @@ using System; -using Aki.Reflection.Patching; -using Diz.DependencyManager; -using UnityEngine.Build.Pipeline; using System.IO; using System.Linq; using System.Reflection; +using Diz.DependencyManager; +using UnityEngine.Build.Pipeline; using Aki.Custom.Models; using Aki.Custom.Utils; +using Aki.Reflection.Patching; namespace Aki.Custom.Patches { @@ -38,7 +38,7 @@ namespace Aki.Custom.Patches : bundle.Dependencies; // set path to either cache (HTTP) or mod (local) - filepath = BundleManager.GetBundlePath(bundle); + filepath = BundleManager.GetBundleFilePath(bundle); } _ = new EasyBundleHelper(__instance) diff --git a/project/Aki.Custom/Utils/BundleManager.cs b/project/Aki.Custom/Utils/BundleManager.cs index c2e13a9..a868bd3 100644 --- a/project/Aki.Custom/Utils/BundleManager.cs +++ b/project/Aki.Custom/Utils/BundleManager.cs @@ -10,74 +10,51 @@ namespace Aki.Custom.Utils { public static class BundleManager { + private const string CachePath = "user/cache/bundles/"; private static readonly ManualLogSource _logger; public static readonly ConcurrentDictionary Bundles; - public static string CachePath; static BundleManager() { _logger = Logger.CreateLogSource(nameof(BundleManager)); Bundles = new ConcurrentDictionary(); - CachePath = "user/cache/bundles/"; } public static string GetBundlePath(BundleItem bundle) { return RequestHandler.IsLocal ? $"{bundle.ModPath}/bundles/{bundle.FileName}" - : CachePath + bundle.FileName; + : CachePath; } - public static async Task GetBundles() + public static string GetBundleFilePath(BundleItem bundle) + { + return GetBundlePath(bundle) + bundle.FileName; + } + + public static async Task DownloadManifest() { // get bundles - var json = RequestHandler.GetJson("/singleplayer/bundles"); + var json = await RequestHandler.GetJsonAsync("/singleplayer/bundles"); var bundles = JsonConvert.DeserializeObject(json); - // register bundles - var toDownload = new ConcurrentBag(); - foreach (var bundle in bundles) { Bundles.TryAdd(bundle.FileName, bundle); - - if (await ShouldReaquire(bundle)) - { - // mark for download - toDownload.Add(bundle); - } - } - - if (RequestHandler.IsLocal) - { - // loading from local mods - _logger.LogInfo("CACHE: Loading all bundles from mods on disk."); - return; - } - else - { - // download bundles - // NOTE: assumes bundle keys to be unique - foreach (var bundle in toDownload) - { - // download bundle - var filepath = GetBundlePath(bundle); - var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); - await VFS.WriteFileAsync(filepath, data); - } } } - private static async Task ShouldReaquire(BundleItem bundle) + public static async Task DownloadBundle(BundleItem bundle) { - if (RequestHandler.IsLocal) - { - // only handle remote bundles - return false; - } + var filepath = GetBundleFilePath(bundle); + var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); + await VFS.WriteFileAsync(filepath, data); + } + public static async Task ShouldReaquire(BundleItem bundle) + { // read cache - var filepath = CachePath + bundle.FileName; + var filepath = GetBundleFilePath(bundle); if (VFS.Exists(filepath)) { @@ -88,7 +65,11 @@ namespace Aki.Custom.Utils if (crc == bundle.Crc) { // file is up-to-date - _logger.LogInfo($"CACHE: Loading locally {bundle.FileName}"); + var location = RequestHandler.IsLocal + ? "MOD" + : "CACHE"; + + _logger.LogInfo($"{location}: Loading locally {bundle.FileName}"); return false; } else From 64296e3e62e92cc1b4a77f91384e579008872d95 Mon Sep 17 00:00:00 2001 From: Dev Date: Sun, 12 May 2024 23:03:30 +0100 Subject: [PATCH 3/5] Revert "Improve async bundles (!123)" This reverts commit 9e65e68c81f47b05b29244f08d3cb2b9edc8616d. --- project/Aki.Custom/Patches/EasyAssetsPatch.cs | 67 +++++++++---------- project/Aki.Custom/Patches/EasyBundlePatch.cs | 8 +-- project/Aki.Custom/Utils/BundleManager.cs | 63 +++++++++++------ 3 files changed, 75 insertions(+), 63 deletions(-) diff --git a/project/Aki.Custom/Patches/EasyAssetsPatch.cs b/project/Aki.Custom/Patches/EasyAssetsPatch.cs index c62465c..3ebeb97 100644 --- a/project/Aki.Custom/Patches/EasyAssetsPatch.cs +++ b/project/Aki.Custom/Patches/EasyAssetsPatch.cs @@ -1,19 +1,21 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; +using Aki.Reflection.Patching; +using Diz.Jobs; using Diz.Resources; using JetBrains.Annotations; using Newtonsoft.Json; using UnityEngine; using UnityEngine.Build.Pipeline; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; using Aki.Common.Utils; using Aki.Custom.Models; using Aki.Custom.Utils; -using Aki.Reflection.Patching; -using Aki.Reflection.Utils; using DependencyGraph = DependencyGraph; +using Aki.Reflection.Utils; namespace Aki.Custom.Patches { @@ -36,33 +38,40 @@ namespace Aki.Custom.Patches protected override MethodBase GetTargetMethod() { - return typeof(EasyAssets).GetMethod(nameof(EasyAssets.Create)); + return typeof(EasyAssets).GetMethods(PatchConstants.PublicDeclaredFlags).SingleCustom(IsTargetMethod); + } + + private static bool IsTargetMethod(MethodInfo mi) + { + var parameters = mi.GetParameters(); + return (parameters.Length == 6 + && parameters[0].Name == "bundleLock" + && parameters[1].Name == "defaultKey" + && parameters[4].Name == "shouldExclude"); } [PatchPrefix] - private static bool PatchPrefix(ref Task __result, GameObject gameObject, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, + private static bool PatchPrefix(ref Task __result, EasyAssets __instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, [CanBeNull] Func bundleCheck) { - var easyAsset = gameObject.AddComponent(); - __result = Init(easyAsset, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); - + __result = Init(__instance, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); return false; } - private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) + private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) { // platform manifest - var eftBundlesPath = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; - var filepath = eftBundlesPath + platformName; + var path = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; + var filepath = path + platformName; var jsonfile = filepath + ".json"; - var manifest = VFS.Exists(jsonfile) + var manifest = File.Exists(jsonfile) ? await GetManifestJson(jsonfile) : await GetManifestBundle(filepath); // lazy-initialize aki bundles if (BundleManager.Bundles.Keys.Count == 0) { - await BundleManager.DownloadManifest(); + await BundleManager.GetBundles(); } // create bundles array from obfuscated type @@ -76,43 +85,27 @@ namespace Aki.Custom.Patches bundleLock = new BundleLock(int.MaxValue); } + // create bundle of obfuscated type var bundles = (IEasyBundle[])Array.CreateInstance(EasyBundleHelper.Type, bundleNames.Length); for (var i = 0; i < bundleNames.Length; i++) { - var key = bundleNames[i]; - var path = eftBundlesPath; - - // acquire external bundle - if (BundleManager.Bundles.TryGetValue(key, out var bundleInfo)) - { - // we need base path without file extension - path = BundleManager.GetBundlePath(bundleInfo); - - // only download when connected externally - if (await BundleManager.ShouldReaquire(bundleInfo)) - { - await BundleManager.DownloadBundle(bundleInfo); - } - } - - // create bundle of obfuscated type bundles[i] = (IEasyBundle)Activator.CreateInstance(EasyBundleHelper.Type, new object[] { - key, + bundleNames[i], path, manifest, bundleLock, bundleCheck }); + + await JobScheduler.Yield(EJobPriority.Immediate); } // create dependency graph instance.Manifest = manifest; _bundlesField.SetValue(instance, bundles); instance.System = new DependencyGraph(bundles, defaultKey, shouldExclude); - - return instance; } // NOTE: used by: diff --git a/project/Aki.Custom/Patches/EasyBundlePatch.cs b/project/Aki.Custom/Patches/EasyBundlePatch.cs index b95593d..f5e727c 100644 --- a/project/Aki.Custom/Patches/EasyBundlePatch.cs +++ b/project/Aki.Custom/Patches/EasyBundlePatch.cs @@ -1,12 +1,12 @@ using System; +using Aki.Reflection.Patching; +using Diz.DependencyManager; +using UnityEngine.Build.Pipeline; using System.IO; using System.Linq; using System.Reflection; -using Diz.DependencyManager; -using UnityEngine.Build.Pipeline; using Aki.Custom.Models; using Aki.Custom.Utils; -using Aki.Reflection.Patching; namespace Aki.Custom.Patches { @@ -38,7 +38,7 @@ namespace Aki.Custom.Patches : bundle.Dependencies; // set path to either cache (HTTP) or mod (local) - filepath = BundleManager.GetBundleFilePath(bundle); + filepath = BundleManager.GetBundlePath(bundle); } _ = new EasyBundleHelper(__instance) diff --git a/project/Aki.Custom/Utils/BundleManager.cs b/project/Aki.Custom/Utils/BundleManager.cs index a868bd3..c2e13a9 100644 --- a/project/Aki.Custom/Utils/BundleManager.cs +++ b/project/Aki.Custom/Utils/BundleManager.cs @@ -10,51 +10,74 @@ namespace Aki.Custom.Utils { public static class BundleManager { - private const string CachePath = "user/cache/bundles/"; private static readonly ManualLogSource _logger; public static readonly ConcurrentDictionary Bundles; + public static string CachePath; static BundleManager() { _logger = Logger.CreateLogSource(nameof(BundleManager)); Bundles = new ConcurrentDictionary(); + CachePath = "user/cache/bundles/"; } public static string GetBundlePath(BundleItem bundle) { return RequestHandler.IsLocal ? $"{bundle.ModPath}/bundles/{bundle.FileName}" - : CachePath; + : CachePath + bundle.FileName; } - public static string GetBundleFilePath(BundleItem bundle) - { - return GetBundlePath(bundle) + bundle.FileName; - } - - public static async Task DownloadManifest() + public static async Task GetBundles() { // get bundles - var json = await RequestHandler.GetJsonAsync("/singleplayer/bundles"); + var json = RequestHandler.GetJson("/singleplayer/bundles"); var bundles = JsonConvert.DeserializeObject(json); + // register bundles + var toDownload = new ConcurrentBag(); + foreach (var bundle in bundles) { Bundles.TryAdd(bundle.FileName, bundle); + + if (await ShouldReaquire(bundle)) + { + // mark for download + toDownload.Add(bundle); + } + } + + if (RequestHandler.IsLocal) + { + // loading from local mods + _logger.LogInfo("CACHE: Loading all bundles from mods on disk."); + return; + } + else + { + // download bundles + // NOTE: assumes bundle keys to be unique + foreach (var bundle in toDownload) + { + // download bundle + var filepath = GetBundlePath(bundle); + var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); + await VFS.WriteFileAsync(filepath, data); + } } } - public static async Task DownloadBundle(BundleItem bundle) + private static async Task ShouldReaquire(BundleItem bundle) { - var filepath = GetBundleFilePath(bundle); - var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); - await VFS.WriteFileAsync(filepath, data); - } + if (RequestHandler.IsLocal) + { + // only handle remote bundles + return false; + } - public static async Task ShouldReaquire(BundleItem bundle) - { // read cache - var filepath = GetBundleFilePath(bundle); + var filepath = CachePath + bundle.FileName; if (VFS.Exists(filepath)) { @@ -65,11 +88,7 @@ namespace Aki.Custom.Utils if (crc == bundle.Crc) { // file is up-to-date - var location = RequestHandler.IsLocal - ? "MOD" - : "CACHE"; - - _logger.LogInfo($"{location}: Loading locally {bundle.FileName}"); + _logger.LogInfo($"CACHE: Loading locally {bundle.FileName}"); return false; } else From 336ad97bc87b0fdf37a5540f646b79c7364948a8 Mon Sep 17 00:00:00 2001 From: Dev Date: Sun, 12 May 2024 23:22:37 +0100 Subject: [PATCH 4/5] Revert "Revert "Improve async bundles (!123)"" This reverts commit 64296e3e62e92cc1b4a77f91384e579008872d95. --- project/Aki.Custom/Patches/EasyAssetsPatch.cs | 67 ++++++++++--------- project/Aki.Custom/Patches/EasyBundlePatch.cs | 8 +-- project/Aki.Custom/Utils/BundleManager.cs | 63 ++++++----------- 3 files changed, 63 insertions(+), 75 deletions(-) diff --git a/project/Aki.Custom/Patches/EasyAssetsPatch.cs b/project/Aki.Custom/Patches/EasyAssetsPatch.cs index 3ebeb97..c62465c 100644 --- a/project/Aki.Custom/Patches/EasyAssetsPatch.cs +++ b/project/Aki.Custom/Patches/EasyAssetsPatch.cs @@ -1,21 +1,19 @@ -using Aki.Reflection.Patching; -using Diz.Jobs; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; using Diz.Resources; using JetBrains.Annotations; using Newtonsoft.Json; using UnityEngine; using UnityEngine.Build.Pipeline; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; using Aki.Common.Utils; using Aki.Custom.Models; using Aki.Custom.Utils; -using DependencyGraph = DependencyGraph; +using Aki.Reflection.Patching; using Aki.Reflection.Utils; +using DependencyGraph = DependencyGraph; namespace Aki.Custom.Patches { @@ -38,40 +36,33 @@ namespace Aki.Custom.Patches protected override MethodBase GetTargetMethod() { - return typeof(EasyAssets).GetMethods(PatchConstants.PublicDeclaredFlags).SingleCustom(IsTargetMethod); - } - - private static bool IsTargetMethod(MethodInfo mi) - { - var parameters = mi.GetParameters(); - return (parameters.Length == 6 - && parameters[0].Name == "bundleLock" - && parameters[1].Name == "defaultKey" - && parameters[4].Name == "shouldExclude"); + return typeof(EasyAssets).GetMethod(nameof(EasyAssets.Create)); } [PatchPrefix] - private static bool PatchPrefix(ref Task __result, EasyAssets __instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, + private static bool PatchPrefix(ref Task __result, GameObject gameObject, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, [CanBeNull] Func bundleCheck) { - __result = Init(__instance, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); + var easyAsset = gameObject.AddComponent(); + __result = Init(easyAsset, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); + return false; } - private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) + private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) { // platform manifest - var path = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; - var filepath = path + platformName; + var eftBundlesPath = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; + var filepath = eftBundlesPath + platformName; var jsonfile = filepath + ".json"; - var manifest = File.Exists(jsonfile) + var manifest = VFS.Exists(jsonfile) ? await GetManifestJson(jsonfile) : await GetManifestBundle(filepath); // lazy-initialize aki bundles if (BundleManager.Bundles.Keys.Count == 0) { - await BundleManager.GetBundles(); + await BundleManager.DownloadManifest(); } // create bundles array from obfuscated type @@ -85,27 +76,43 @@ namespace Aki.Custom.Patches bundleLock = new BundleLock(int.MaxValue); } - // create bundle of obfuscated type var bundles = (IEasyBundle[])Array.CreateInstance(EasyBundleHelper.Type, bundleNames.Length); for (var i = 0; i < bundleNames.Length; i++) { + var key = bundleNames[i]; + var path = eftBundlesPath; + + // acquire external bundle + if (BundleManager.Bundles.TryGetValue(key, out var bundleInfo)) + { + // we need base path without file extension + path = BundleManager.GetBundlePath(bundleInfo); + + // only download when connected externally + if (await BundleManager.ShouldReaquire(bundleInfo)) + { + await BundleManager.DownloadBundle(bundleInfo); + } + } + + // create bundle of obfuscated type bundles[i] = (IEasyBundle)Activator.CreateInstance(EasyBundleHelper.Type, new object[] { - bundleNames[i], + key, path, manifest, bundleLock, bundleCheck }); - - await JobScheduler.Yield(EJobPriority.Immediate); } // create dependency graph instance.Manifest = manifest; _bundlesField.SetValue(instance, bundles); instance.System = new DependencyGraph(bundles, defaultKey, shouldExclude); + + return instance; } // NOTE: used by: diff --git a/project/Aki.Custom/Patches/EasyBundlePatch.cs b/project/Aki.Custom/Patches/EasyBundlePatch.cs index f5e727c..b95593d 100644 --- a/project/Aki.Custom/Patches/EasyBundlePatch.cs +++ b/project/Aki.Custom/Patches/EasyBundlePatch.cs @@ -1,12 +1,12 @@ using System; -using Aki.Reflection.Patching; -using Diz.DependencyManager; -using UnityEngine.Build.Pipeline; using System.IO; using System.Linq; using System.Reflection; +using Diz.DependencyManager; +using UnityEngine.Build.Pipeline; using Aki.Custom.Models; using Aki.Custom.Utils; +using Aki.Reflection.Patching; namespace Aki.Custom.Patches { @@ -38,7 +38,7 @@ namespace Aki.Custom.Patches : bundle.Dependencies; // set path to either cache (HTTP) or mod (local) - filepath = BundleManager.GetBundlePath(bundle); + filepath = BundleManager.GetBundleFilePath(bundle); } _ = new EasyBundleHelper(__instance) diff --git a/project/Aki.Custom/Utils/BundleManager.cs b/project/Aki.Custom/Utils/BundleManager.cs index c2e13a9..a868bd3 100644 --- a/project/Aki.Custom/Utils/BundleManager.cs +++ b/project/Aki.Custom/Utils/BundleManager.cs @@ -10,74 +10,51 @@ namespace Aki.Custom.Utils { public static class BundleManager { + private const string CachePath = "user/cache/bundles/"; private static readonly ManualLogSource _logger; public static readonly ConcurrentDictionary Bundles; - public static string CachePath; static BundleManager() { _logger = Logger.CreateLogSource(nameof(BundleManager)); Bundles = new ConcurrentDictionary(); - CachePath = "user/cache/bundles/"; } public static string GetBundlePath(BundleItem bundle) { return RequestHandler.IsLocal ? $"{bundle.ModPath}/bundles/{bundle.FileName}" - : CachePath + bundle.FileName; + : CachePath; } - public static async Task GetBundles() + public static string GetBundleFilePath(BundleItem bundle) + { + return GetBundlePath(bundle) + bundle.FileName; + } + + public static async Task DownloadManifest() { // get bundles - var json = RequestHandler.GetJson("/singleplayer/bundles"); + var json = await RequestHandler.GetJsonAsync("/singleplayer/bundles"); var bundles = JsonConvert.DeserializeObject(json); - // register bundles - var toDownload = new ConcurrentBag(); - foreach (var bundle in bundles) { Bundles.TryAdd(bundle.FileName, bundle); - - if (await ShouldReaquire(bundle)) - { - // mark for download - toDownload.Add(bundle); - } - } - - if (RequestHandler.IsLocal) - { - // loading from local mods - _logger.LogInfo("CACHE: Loading all bundles from mods on disk."); - return; - } - else - { - // download bundles - // NOTE: assumes bundle keys to be unique - foreach (var bundle in toDownload) - { - // download bundle - var filepath = GetBundlePath(bundle); - var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); - await VFS.WriteFileAsync(filepath, data); - } } } - private static async Task ShouldReaquire(BundleItem bundle) + public static async Task DownloadBundle(BundleItem bundle) { - if (RequestHandler.IsLocal) - { - // only handle remote bundles - return false; - } + var filepath = GetBundleFilePath(bundle); + var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); + await VFS.WriteFileAsync(filepath, data); + } + public static async Task ShouldReaquire(BundleItem bundle) + { // read cache - var filepath = CachePath + bundle.FileName; + var filepath = GetBundleFilePath(bundle); if (VFS.Exists(filepath)) { @@ -88,7 +65,11 @@ namespace Aki.Custom.Utils if (crc == bundle.Crc) { // file is up-to-date - _logger.LogInfo($"CACHE: Loading locally {bundle.FileName}"); + var location = RequestHandler.IsLocal + ? "MOD" + : "CACHE"; + + _logger.LogInfo($"{location}: Loading locally {bundle.FileName}"); return false; } else From 14079619cfa7fe130f42b9ca56e650830db0c879 Mon Sep 17 00:00:00 2001 From: Dev Date: Mon, 13 May 2024 09:39:48 +0100 Subject: [PATCH 5/5] Fixed bundle path issue --- project/Aki.Custom/Utils/BundleManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Aki.Custom/Utils/BundleManager.cs b/project/Aki.Custom/Utils/BundleManager.cs index a868bd3..b49f227 100644 --- a/project/Aki.Custom/Utils/BundleManager.cs +++ b/project/Aki.Custom/Utils/BundleManager.cs @@ -23,7 +23,7 @@ namespace Aki.Custom.Utils public static string GetBundlePath(BundleItem bundle) { return RequestHandler.IsLocal - ? $"{bundle.ModPath}/bundles/{bundle.FileName}" + ? $"{bundle.ModPath}/bundles/" : CachePath; }