From 64296e3e62e92cc1b4a77f91384e579008872d95 Mon Sep 17 00:00:00 2001 From: Dev Date: Sun, 12 May 2024 23:03:30 +0100 Subject: [PATCH 1/3] Revert "Improve async bundles (!123)" This reverts commit 9e65e68c81f47b05b29244f08d3cb2b9edc8616d. --- project/Aki.Custom/Patches/EasyAssetsPatch.cs | 67 +++++++++---------- project/Aki.Custom/Patches/EasyBundlePatch.cs | 8 +-- project/Aki.Custom/Utils/BundleManager.cs | 63 +++++++++++------ 3 files changed, 75 insertions(+), 63 deletions(-) diff --git a/project/Aki.Custom/Patches/EasyAssetsPatch.cs b/project/Aki.Custom/Patches/EasyAssetsPatch.cs index c62465c..3ebeb97 100644 --- a/project/Aki.Custom/Patches/EasyAssetsPatch.cs +++ b/project/Aki.Custom/Patches/EasyAssetsPatch.cs @@ -1,19 +1,21 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; +using Aki.Reflection.Patching; +using Diz.Jobs; using Diz.Resources; using JetBrains.Annotations; using Newtonsoft.Json; using UnityEngine; using UnityEngine.Build.Pipeline; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; using Aki.Common.Utils; using Aki.Custom.Models; using Aki.Custom.Utils; -using Aki.Reflection.Patching; -using Aki.Reflection.Utils; using DependencyGraph = DependencyGraph; +using Aki.Reflection.Utils; namespace Aki.Custom.Patches { @@ -36,33 +38,40 @@ namespace Aki.Custom.Patches protected override MethodBase GetTargetMethod() { - return typeof(EasyAssets).GetMethod(nameof(EasyAssets.Create)); + return typeof(EasyAssets).GetMethods(PatchConstants.PublicDeclaredFlags).SingleCustom(IsTargetMethod); + } + + private static bool IsTargetMethod(MethodInfo mi) + { + var parameters = mi.GetParameters(); + return (parameters.Length == 6 + && parameters[0].Name == "bundleLock" + && parameters[1].Name == "defaultKey" + && parameters[4].Name == "shouldExclude"); } [PatchPrefix] - private static bool PatchPrefix(ref Task __result, GameObject gameObject, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, + private static bool PatchPrefix(ref Task __result, EasyAssets __instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, [CanBeNull] Func bundleCheck) { - var easyAsset = gameObject.AddComponent(); - __result = Init(easyAsset, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); - + __result = Init(__instance, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); return false; } - private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) + private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) { // platform manifest - var eftBundlesPath = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; - var filepath = eftBundlesPath + platformName; + var path = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; + var filepath = path + platformName; var jsonfile = filepath + ".json"; - var manifest = VFS.Exists(jsonfile) + var manifest = File.Exists(jsonfile) ? await GetManifestJson(jsonfile) : await GetManifestBundle(filepath); // lazy-initialize aki bundles if (BundleManager.Bundles.Keys.Count == 0) { - await BundleManager.DownloadManifest(); + await BundleManager.GetBundles(); } // create bundles array from obfuscated type @@ -76,43 +85,27 @@ namespace Aki.Custom.Patches bundleLock = new BundleLock(int.MaxValue); } + // create bundle of obfuscated type var bundles = (IEasyBundle[])Array.CreateInstance(EasyBundleHelper.Type, bundleNames.Length); for (var i = 0; i < bundleNames.Length; i++) { - var key = bundleNames[i]; - var path = eftBundlesPath; - - // acquire external bundle - if (BundleManager.Bundles.TryGetValue(key, out var bundleInfo)) - { - // we need base path without file extension - path = BundleManager.GetBundlePath(bundleInfo); - - // only download when connected externally - if (await BundleManager.ShouldReaquire(bundleInfo)) - { - await BundleManager.DownloadBundle(bundleInfo); - } - } - - // create bundle of obfuscated type bundles[i] = (IEasyBundle)Activator.CreateInstance(EasyBundleHelper.Type, new object[] { - key, + bundleNames[i], path, manifest, bundleLock, bundleCheck }); + + await JobScheduler.Yield(EJobPriority.Immediate); } // create dependency graph instance.Manifest = manifest; _bundlesField.SetValue(instance, bundles); instance.System = new DependencyGraph(bundles, defaultKey, shouldExclude); - - return instance; } // NOTE: used by: diff --git a/project/Aki.Custom/Patches/EasyBundlePatch.cs b/project/Aki.Custom/Patches/EasyBundlePatch.cs index b95593d..f5e727c 100644 --- a/project/Aki.Custom/Patches/EasyBundlePatch.cs +++ b/project/Aki.Custom/Patches/EasyBundlePatch.cs @@ -1,12 +1,12 @@ using System; +using Aki.Reflection.Patching; +using Diz.DependencyManager; +using UnityEngine.Build.Pipeline; using System.IO; using System.Linq; using System.Reflection; -using Diz.DependencyManager; -using UnityEngine.Build.Pipeline; using Aki.Custom.Models; using Aki.Custom.Utils; -using Aki.Reflection.Patching; namespace Aki.Custom.Patches { @@ -38,7 +38,7 @@ namespace Aki.Custom.Patches : bundle.Dependencies; // set path to either cache (HTTP) or mod (local) - filepath = BundleManager.GetBundleFilePath(bundle); + filepath = BundleManager.GetBundlePath(bundle); } _ = new EasyBundleHelper(__instance) diff --git a/project/Aki.Custom/Utils/BundleManager.cs b/project/Aki.Custom/Utils/BundleManager.cs index a868bd3..c2e13a9 100644 --- a/project/Aki.Custom/Utils/BundleManager.cs +++ b/project/Aki.Custom/Utils/BundleManager.cs @@ -10,51 +10,74 @@ namespace Aki.Custom.Utils { public static class BundleManager { - private const string CachePath = "user/cache/bundles/"; private static readonly ManualLogSource _logger; public static readonly ConcurrentDictionary Bundles; + public static string CachePath; static BundleManager() { _logger = Logger.CreateLogSource(nameof(BundleManager)); Bundles = new ConcurrentDictionary(); + CachePath = "user/cache/bundles/"; } public static string GetBundlePath(BundleItem bundle) { return RequestHandler.IsLocal ? $"{bundle.ModPath}/bundles/{bundle.FileName}" - : CachePath; + : CachePath + bundle.FileName; } - public static string GetBundleFilePath(BundleItem bundle) - { - return GetBundlePath(bundle) + bundle.FileName; - } - - public static async Task DownloadManifest() + public static async Task GetBundles() { // get bundles - var json = await RequestHandler.GetJsonAsync("/singleplayer/bundles"); + var json = RequestHandler.GetJson("/singleplayer/bundles"); var bundles = JsonConvert.DeserializeObject(json); + // register bundles + var toDownload = new ConcurrentBag(); + foreach (var bundle in bundles) { Bundles.TryAdd(bundle.FileName, bundle); + + if (await ShouldReaquire(bundle)) + { + // mark for download + toDownload.Add(bundle); + } + } + + if (RequestHandler.IsLocal) + { + // loading from local mods + _logger.LogInfo("CACHE: Loading all bundles from mods on disk."); + return; + } + else + { + // download bundles + // NOTE: assumes bundle keys to be unique + foreach (var bundle in toDownload) + { + // download bundle + var filepath = GetBundlePath(bundle); + var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); + await VFS.WriteFileAsync(filepath, data); + } } } - public static async Task DownloadBundle(BundleItem bundle) + private static async Task ShouldReaquire(BundleItem bundle) { - var filepath = GetBundleFilePath(bundle); - var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); - await VFS.WriteFileAsync(filepath, data); - } + if (RequestHandler.IsLocal) + { + // only handle remote bundles + return false; + } - public static async Task ShouldReaquire(BundleItem bundle) - { // read cache - var filepath = GetBundleFilePath(bundle); + var filepath = CachePath + bundle.FileName; if (VFS.Exists(filepath)) { @@ -65,11 +88,7 @@ namespace Aki.Custom.Utils if (crc == bundle.Crc) { // file is up-to-date - var location = RequestHandler.IsLocal - ? "MOD" - : "CACHE"; - - _logger.LogInfo($"{location}: Loading locally {bundle.FileName}"); + _logger.LogInfo($"CACHE: Loading locally {bundle.FileName}"); return false; } else From 336ad97bc87b0fdf37a5540f646b79c7364948a8 Mon Sep 17 00:00:00 2001 From: Dev Date: Sun, 12 May 2024 23:22:37 +0100 Subject: [PATCH 2/3] Revert "Revert "Improve async bundles (!123)"" This reverts commit 64296e3e62e92cc1b4a77f91384e579008872d95. --- project/Aki.Custom/Patches/EasyAssetsPatch.cs | 67 ++++++++++--------- project/Aki.Custom/Patches/EasyBundlePatch.cs | 8 +-- project/Aki.Custom/Utils/BundleManager.cs | 63 ++++++----------- 3 files changed, 63 insertions(+), 75 deletions(-) diff --git a/project/Aki.Custom/Patches/EasyAssetsPatch.cs b/project/Aki.Custom/Patches/EasyAssetsPatch.cs index 3ebeb97..c62465c 100644 --- a/project/Aki.Custom/Patches/EasyAssetsPatch.cs +++ b/project/Aki.Custom/Patches/EasyAssetsPatch.cs @@ -1,21 +1,19 @@ -using Aki.Reflection.Patching; -using Diz.Jobs; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; using Diz.Resources; using JetBrains.Annotations; using Newtonsoft.Json; using UnityEngine; using UnityEngine.Build.Pipeline; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; using Aki.Common.Utils; using Aki.Custom.Models; using Aki.Custom.Utils; -using DependencyGraph = DependencyGraph; +using Aki.Reflection.Patching; using Aki.Reflection.Utils; +using DependencyGraph = DependencyGraph; namespace Aki.Custom.Patches { @@ -38,40 +36,33 @@ namespace Aki.Custom.Patches protected override MethodBase GetTargetMethod() { - return typeof(EasyAssets).GetMethods(PatchConstants.PublicDeclaredFlags).SingleCustom(IsTargetMethod); - } - - private static bool IsTargetMethod(MethodInfo mi) - { - var parameters = mi.GetParameters(); - return (parameters.Length == 6 - && parameters[0].Name == "bundleLock" - && parameters[1].Name == "defaultKey" - && parameters[4].Name == "shouldExclude"); + return typeof(EasyAssets).GetMethod(nameof(EasyAssets.Create)); } [PatchPrefix] - private static bool PatchPrefix(ref Task __result, EasyAssets __instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, + private static bool PatchPrefix(ref Task __result, GameObject gameObject, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, [CanBeNull] Func bundleCheck) { - __result = Init(__instance, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); + var easyAsset = gameObject.AddComponent(); + __result = Init(easyAsset, bundleLock, defaultKey, rootPath, platformName, shouldExclude, bundleCheck); + return false; } - private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) + private static async Task Init(EasyAssets instance, [CanBeNull] IBundleLock bundleLock, string defaultKey, string rootPath, string platformName, [CanBeNull] Func shouldExclude, Func bundleCheck) { // platform manifest - var path = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; - var filepath = path + platformName; + var eftBundlesPath = $"{rootPath.Replace("file:///", string.Empty).Replace("file://", string.Empty)}/{platformName}/"; + var filepath = eftBundlesPath + platformName; var jsonfile = filepath + ".json"; - var manifest = File.Exists(jsonfile) + var manifest = VFS.Exists(jsonfile) ? await GetManifestJson(jsonfile) : await GetManifestBundle(filepath); // lazy-initialize aki bundles if (BundleManager.Bundles.Keys.Count == 0) { - await BundleManager.GetBundles(); + await BundleManager.DownloadManifest(); } // create bundles array from obfuscated type @@ -85,27 +76,43 @@ namespace Aki.Custom.Patches bundleLock = new BundleLock(int.MaxValue); } - // create bundle of obfuscated type var bundles = (IEasyBundle[])Array.CreateInstance(EasyBundleHelper.Type, bundleNames.Length); for (var i = 0; i < bundleNames.Length; i++) { + var key = bundleNames[i]; + var path = eftBundlesPath; + + // acquire external bundle + if (BundleManager.Bundles.TryGetValue(key, out var bundleInfo)) + { + // we need base path without file extension + path = BundleManager.GetBundlePath(bundleInfo); + + // only download when connected externally + if (await BundleManager.ShouldReaquire(bundleInfo)) + { + await BundleManager.DownloadBundle(bundleInfo); + } + } + + // create bundle of obfuscated type bundles[i] = (IEasyBundle)Activator.CreateInstance(EasyBundleHelper.Type, new object[] { - bundleNames[i], + key, path, manifest, bundleLock, bundleCheck }); - - await JobScheduler.Yield(EJobPriority.Immediate); } // create dependency graph instance.Manifest = manifest; _bundlesField.SetValue(instance, bundles); instance.System = new DependencyGraph(bundles, defaultKey, shouldExclude); + + return instance; } // NOTE: used by: diff --git a/project/Aki.Custom/Patches/EasyBundlePatch.cs b/project/Aki.Custom/Patches/EasyBundlePatch.cs index f5e727c..b95593d 100644 --- a/project/Aki.Custom/Patches/EasyBundlePatch.cs +++ b/project/Aki.Custom/Patches/EasyBundlePatch.cs @@ -1,12 +1,12 @@ using System; -using Aki.Reflection.Patching; -using Diz.DependencyManager; -using UnityEngine.Build.Pipeline; using System.IO; using System.Linq; using System.Reflection; +using Diz.DependencyManager; +using UnityEngine.Build.Pipeline; using Aki.Custom.Models; using Aki.Custom.Utils; +using Aki.Reflection.Patching; namespace Aki.Custom.Patches { @@ -38,7 +38,7 @@ namespace Aki.Custom.Patches : bundle.Dependencies; // set path to either cache (HTTP) or mod (local) - filepath = BundleManager.GetBundlePath(bundle); + filepath = BundleManager.GetBundleFilePath(bundle); } _ = new EasyBundleHelper(__instance) diff --git a/project/Aki.Custom/Utils/BundleManager.cs b/project/Aki.Custom/Utils/BundleManager.cs index c2e13a9..a868bd3 100644 --- a/project/Aki.Custom/Utils/BundleManager.cs +++ b/project/Aki.Custom/Utils/BundleManager.cs @@ -10,74 +10,51 @@ namespace Aki.Custom.Utils { public static class BundleManager { + private const string CachePath = "user/cache/bundles/"; private static readonly ManualLogSource _logger; public static readonly ConcurrentDictionary Bundles; - public static string CachePath; static BundleManager() { _logger = Logger.CreateLogSource(nameof(BundleManager)); Bundles = new ConcurrentDictionary(); - CachePath = "user/cache/bundles/"; } public static string GetBundlePath(BundleItem bundle) { return RequestHandler.IsLocal ? $"{bundle.ModPath}/bundles/{bundle.FileName}" - : CachePath + bundle.FileName; + : CachePath; } - public static async Task GetBundles() + public static string GetBundleFilePath(BundleItem bundle) + { + return GetBundlePath(bundle) + bundle.FileName; + } + + public static async Task DownloadManifest() { // get bundles - var json = RequestHandler.GetJson("/singleplayer/bundles"); + var json = await RequestHandler.GetJsonAsync("/singleplayer/bundles"); var bundles = JsonConvert.DeserializeObject(json); - // register bundles - var toDownload = new ConcurrentBag(); - foreach (var bundle in bundles) { Bundles.TryAdd(bundle.FileName, bundle); - - if (await ShouldReaquire(bundle)) - { - // mark for download - toDownload.Add(bundle); - } - } - - if (RequestHandler.IsLocal) - { - // loading from local mods - _logger.LogInfo("CACHE: Loading all bundles from mods on disk."); - return; - } - else - { - // download bundles - // NOTE: assumes bundle keys to be unique - foreach (var bundle in toDownload) - { - // download bundle - var filepath = GetBundlePath(bundle); - var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); - await VFS.WriteFileAsync(filepath, data); - } } } - private static async Task ShouldReaquire(BundleItem bundle) + public static async Task DownloadBundle(BundleItem bundle) { - if (RequestHandler.IsLocal) - { - // only handle remote bundles - return false; - } + var filepath = GetBundleFilePath(bundle); + var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}"); + await VFS.WriteFileAsync(filepath, data); + } + public static async Task ShouldReaquire(BundleItem bundle) + { // read cache - var filepath = CachePath + bundle.FileName; + var filepath = GetBundleFilePath(bundle); if (VFS.Exists(filepath)) { @@ -88,7 +65,11 @@ namespace Aki.Custom.Utils if (crc == bundle.Crc) { // file is up-to-date - _logger.LogInfo($"CACHE: Loading locally {bundle.FileName}"); + var location = RequestHandler.IsLocal + ? "MOD" + : "CACHE"; + + _logger.LogInfo($"{location}: Loading locally {bundle.FileName}"); return false; } else From 14079619cfa7fe130f42b9ca56e650830db0c879 Mon Sep 17 00:00:00 2001 From: Dev Date: Mon, 13 May 2024 09:39:48 +0100 Subject: [PATCH 3/3] Fixed bundle path issue --- project/Aki.Custom/Utils/BundleManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Aki.Custom/Utils/BundleManager.cs b/project/Aki.Custom/Utils/BundleManager.cs index a868bd3..b49f227 100644 --- a/project/Aki.Custom/Utils/BundleManager.cs +++ b/project/Aki.Custom/Utils/BundleManager.cs @@ -23,7 +23,7 @@ namespace Aki.Custom.Utils public static string GetBundlePath(BundleItem bundle) { return RequestHandler.IsLocal - ? $"{bundle.ModPath}/bundles/{bundle.FileName}" + ? $"{bundle.ModPath}/bundles/" : CachePath; }