mirror of
https://github.com/sp-tarkov/modules.git
synced 2025-02-13 02:50:45 -05:00
async-bundles (!117)
This patch contains the following: - Initial async VFS code (for reading / writing files) - Simplified Http Client code - Added async support to Http Client, RequestHandler - Improved RequestHandler logging - Deferred bundle loading to EasyAssetPatch - Make GetManifestJson run async This comes with a number of benefits: - When downloading bundles, it will mention which files succeeded or failed to download - Bundle loading happens in the initial screen, not the white screen - Fixed the issue where bundle loading could break bepinex loading (too long load time) - Modders can now make async http request and read/write files async I removed logging of sessionid inside the RequestHandler for each request, sessionid is already visible from bepinex log startup parameters. At last, sorry for the amount of commits it took. I initially wanted to target the 3.9.0 branch, but decided to use 3.8.1 instead as async request can really help out some mods. Reviewed-on: SPT-AKI/Modules#117 Co-authored-by: Merijn Hendriks <merijn.d.hendriks@gmail.com> Co-committed-by: Merijn Hendriks <merijn.d.hendriks@gmail.com>
This commit is contained in:
Merijn Hendriks
parent
ed5428ed88
commit
4b401e7449
@ -1,14 +1,14 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using Aki.Common.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Aki.Common.Utils;
|
||||
|
||||
namespace Aki.Common.Http
|
||||
{
|
||||
// NOTE: you do not want to dispose this, keep a reference for the lifetime
|
||||
// of the application.
|
||||
// NOTE: cannot be made async due to Unity's limitations.
|
||||
// NOTE: Don't dispose this, keep a reference for the lifetime of the
|
||||
// application.
|
||||
public class Client : IDisposable
|
||||
{
|
||||
protected readonly HttpClient _httpv;
|
||||
@ -22,9 +22,9 @@ namespace Aki.Common.Http
|
||||
_accountId = accountId;
|
||||
_retries = retries;
|
||||
|
||||
var handler = new HttpClientHandler()
|
||||
var handler = new HttpClientHandler
|
||||
{
|
||||
// force setting cookies in header instead of CookieContainer
|
||||
// set cookies in header instead
|
||||
UseCookies = false
|
||||
};
|
||||
|
||||
@ -43,7 +43,7 @@ namespace Aki.Common.Http
|
||||
};
|
||||
}
|
||||
|
||||
protected byte[] Send(HttpMethod method, string path, byte[] data, bool compress = true)
|
||||
protected async Task<byte[]> SendAsync(HttpMethod method, string path, byte[] data, bool zipped = true)
|
||||
{
|
||||
HttpResponseMessage response = null;
|
||||
|
||||
@ -51,17 +51,17 @@ namespace Aki.Common.Http
|
||||
{
|
||||
if (data != null)
|
||||
{
|
||||
// if there is data, convert to payload
|
||||
byte[] payload = (compress)
|
||||
? Zlib.Compress(data, ZlibCompression.Maximum)
|
||||
: data;
|
||||
|
||||
// add payload to request
|
||||
request.Content = new ByteArrayContent(payload);
|
||||
if (zipped)
|
||||
{
|
||||
data = Zlib.Compress(data, ZlibCompression.Maximum);
|
||||
}
|
||||
|
||||
request.Content = new ByteArrayContent(data);
|
||||
}
|
||||
|
||||
// send request
|
||||
response = _httpv.SendAsync(request).Result;
|
||||
response = await _httpv.SendAsync(request);
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
@ -72,85 +72,79 @@ namespace Aki.Common.Http
|
||||
|
||||
using (var ms = new MemoryStream())
|
||||
{
|
||||
using (var stream = response.Content.ReadAsStreamAsync().Result)
|
||||
using (var stream = await response.Content.ReadAsStreamAsync())
|
||||
{
|
||||
// grap response payload
|
||||
stream.CopyTo(ms);
|
||||
var bytes = ms.ToArray();
|
||||
await stream.CopyToAsync(ms);
|
||||
var body = ms.ToArray();
|
||||
|
||||
if (bytes != null)
|
||||
if (Zlib.IsCompressed(body))
|
||||
{
|
||||
// payload contains data
|
||||
return Zlib.IsCompressed(bytes)
|
||||
? Zlib.Decompress(bytes)
|
||||
: bytes;
|
||||
body = Zlib.Decompress(body);
|
||||
}
|
||||
|
||||
if (body == null)
|
||||
{
|
||||
// payload doesn't contains data
|
||||
var code = response.StatusCode.ToString();
|
||||
body = Encoding.UTF8.GetBytes(code);
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// response returned no data
|
||||
return null;
|
||||
protected async Task<byte[]> SendWithRetriesAsync(HttpMethod method, string path, byte[] data, bool compress = true)
|
||||
{
|
||||
var error = new Exception("Internal error");
|
||||
|
||||
// NOTE: <= is intentional. 0 is send, 1/2/3 is retry
|
||||
for (var i = 0; i <= _retries; ++i)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await SendAsync(method, path, data, compress);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
error = ex;
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
public async Task<byte[]> GetAsync(string path)
|
||||
{
|
||||
return await SendWithRetriesAsync(HttpMethod.Get, path, null);
|
||||
}
|
||||
|
||||
public byte[] Get(string path)
|
||||
{
|
||||
var error = new Exception("Internal error");
|
||||
|
||||
// NOTE: <= is intentional, 0 is send, 1,2,3 is retry
|
||||
for (var i = 0; i <= _retries; ++i)
|
||||
{
|
||||
try
|
||||
{
|
||||
return Send(HttpMethod.Get, path, null, false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
error = ex;
|
||||
}
|
||||
return Task.Run(() => GetAsync(path)).Result;
|
||||
}
|
||||
|
||||
throw error;
|
||||
public async Task<byte[]> PostAsync(string path, byte[] data, bool compress = true)
|
||||
{
|
||||
return await SendWithRetriesAsync(HttpMethod.Post, path, data, compress);
|
||||
}
|
||||
|
||||
public byte[] Post(string path, byte[] data, bool compressed = true)
|
||||
public byte[] Post(string path, byte[] data, bool compress = true)
|
||||
{
|
||||
var error = new Exception("Internal error");
|
||||
|
||||
// NOTE: <= is intentional, 0 is send, 1,2,3 is retry
|
||||
for (var i = 0; i <= _retries; ++i)
|
||||
{
|
||||
try
|
||||
{
|
||||
return Send(HttpMethod.Post, path, data, compressed);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
error = ex;
|
||||
}
|
||||
return Task.Run(() => PostAsync(path, data, compress)).Result;
|
||||
}
|
||||
|
||||
throw error;
|
||||
// NOTE: returns status code as bytes
|
||||
public async Task<byte[]> PutAsync(string path, byte[] data, bool compress = true)
|
||||
{
|
||||
return await SendWithRetriesAsync(HttpMethod.Post, path, data, compress);
|
||||
}
|
||||
|
||||
public void Put(string path, byte[] data, bool compressed = true)
|
||||
// NOTE: returns status code as bytes
|
||||
public byte[] Put(string path, byte[] data, bool compress = true)
|
||||
{
|
||||
var error = new Exception("Internal error");
|
||||
|
||||
// NOTE: <= is intentional, 0 is send, 1,2,3 is retry
|
||||
for (var i = 0; i <= _retries; ++i)
|
||||
{
|
||||
try
|
||||
{
|
||||
Send(HttpMethod.Put, path, data, compressed);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
error = ex;
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
return Task.Run(() => PutAsync(path, data, compress)).Result;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
|
||||
@ -43,65 +43,91 @@ namespace Aki.Common.Http
|
||||
HttpClient = new Client(Host, SessionId);
|
||||
}
|
||||
|
||||
private static void ValidateData(byte[] data)
|
||||
private static void ValidateData(string path, byte[] data)
|
||||
{
|
||||
if (data == null)
|
||||
{
|
||||
_logger.LogError($"Request failed, body is null");
|
||||
_logger.LogError($"[REQUEST FAILED] {path}");
|
||||
}
|
||||
|
||||
_logger.LogInfo($"Request was successful");
|
||||
_logger.LogInfo($"[REQUEST SUCCESSFUL] {path}");
|
||||
}
|
||||
|
||||
private static void ValidateJson(string json)
|
||||
private static void ValidateJson(string path, string json)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(json))
|
||||
{
|
||||
_logger.LogError($"Request failed, body is null");
|
||||
_logger.LogError($"[REQUEST FAILED] {path}");
|
||||
}
|
||||
|
||||
_logger.LogInfo($"Request was successful");
|
||||
_logger.LogInfo($"[REQUEST SUCCESSFUL] {path}");
|
||||
}
|
||||
|
||||
public static async Task<byte[]> GetDataAsync(string path)
|
||||
{
|
||||
_logger.LogInfo($"[REQUEST]: {path}");
|
||||
|
||||
var data = await HttpClient.GetAsync(path);
|
||||
|
||||
ValidateData(path, data);
|
||||
return data;
|
||||
}
|
||||
|
||||
public static byte[] GetData(string path)
|
||||
{
|
||||
_logger.LogInfo($"Request GET data: {SessionId}:{path}");
|
||||
return Task.Run(() => GetData(path)).Result;
|
||||
}
|
||||
|
||||
var data = HttpClient.Get(path);
|
||||
public static async Task<string> GetJsonAsync(string path)
|
||||
{
|
||||
_logger.LogInfo($"[REQUEST]: {path}");
|
||||
|
||||
ValidateData(data);
|
||||
return data;
|
||||
var payload = await HttpClient.GetAsync(path);
|
||||
var body = Encoding.UTF8.GetString(payload);
|
||||
|
||||
ValidateJson(path, body);
|
||||
return body;
|
||||
}
|
||||
|
||||
public static string GetJson(string path)
|
||||
{
|
||||
_logger.LogInfo($"Request GET json: {SessionId}:{path}");
|
||||
|
||||
var payload = HttpClient.Get(path);
|
||||
var body = Encoding.UTF8.GetString(payload);
|
||||
|
||||
ValidateJson(body);
|
||||
return body;
|
||||
return Task.Run(() => GetJsonAsync(path)).Result;
|
||||
}
|
||||
|
||||
public static string PostJson(string path, string json)
|
||||
public static string PostJsonAsync(string path, string json)
|
||||
{
|
||||
_logger.LogInfo($"Request POST json: {SessionId}:{path}");
|
||||
_logger.LogInfo($"[REQUEST]: {path}");
|
||||
|
||||
var payload = Encoding.UTF8.GetBytes(json);
|
||||
var data = HttpClient.Post(path, payload);
|
||||
var body = Encoding.UTF8.GetString(data);
|
||||
|
||||
ValidateJson(body);
|
||||
ValidateJson(path, body);
|
||||
return body;
|
||||
}
|
||||
|
||||
public static void PutJson(string path, string json)
|
||||
public static string PostJson(string path, string json)
|
||||
{
|
||||
_logger.LogInfo($"Request PUT json: {SessionId}:{path}");
|
||||
return Task.Run(() => PostJsonAsync(path, json)).Result;
|
||||
}
|
||||
|
||||
// NOTE: returns status code
|
||||
public static async Task<string> PutJsonAsync(string path, string json)
|
||||
{
|
||||
_logger.LogInfo($"[REQUEST]: {path}");
|
||||
|
||||
var payload = Encoding.UTF8.GetBytes(json);
|
||||
HttpClient.Put(path, payload);
|
||||
var data = await HttpClient.PutAsync(path, payload);
|
||||
var body = Encoding.UTF8.GetString(data);
|
||||
|
||||
ValidateJson(path, body);
|
||||
return body;
|
||||
}
|
||||
|
||||
// NOTE: returns status code
|
||||
public static string PutJson(string path, string json)
|
||||
{
|
||||
return Task.Run(() => PutJsonAsync(path, json)).Result;
|
||||
}
|
||||
|
||||
#region DEPRECATED, REMOVE IN 3.8.1
|
||||
@ -120,7 +146,7 @@ namespace Aki.Common.Http
|
||||
var request = new Request();
|
||||
var data = request.Send(url, "GET", null, headers: headers);
|
||||
|
||||
ValidateData(data);
|
||||
ValidateData(url, data);
|
||||
return data;
|
||||
|
||||
}
|
||||
@ -141,7 +167,7 @@ namespace Aki.Common.Http
|
||||
var data = request.Send(url, "GET", headers: headers);
|
||||
var body = Encoding.UTF8.GetString(data);
|
||||
|
||||
ValidateJson(body);
|
||||
ValidateJson(url, body);
|
||||
return body;
|
||||
|
||||
}
|
||||
@ -164,7 +190,7 @@ namespace Aki.Common.Http
|
||||
var data = request.Send(url, "POST", payload, true, mime, headers);
|
||||
var body = Encoding.UTF8.GetString(data);
|
||||
|
||||
ValidateJson(body);
|
||||
ValidateJson(url, body);
|
||||
return body;
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Aki.Common.Utils
|
||||
{
|
||||
@ -97,6 +98,22 @@ namespace Aki.Common.Utils
|
||||
return File.ReadAllBytes(filepath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get file content as bytes.
|
||||
/// </summary>
|
||||
public static async Task<byte[]> ReadFileAsync(string filepath)
|
||||
{
|
||||
byte[] result;
|
||||
|
||||
using (var fs = File.Open(filepath, FileMode.Open))
|
||||
{
|
||||
result = new byte[fs.Length];
|
||||
await fs.ReadAsync(result, 0, (int)fs.Length);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get file content as string.
|
||||
/// </summary>
|
||||
@ -105,6 +122,20 @@ namespace Aki.Common.Utils
|
||||
return File.ReadAllText(filepath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get file content as string.
|
||||
/// </summary>
|
||||
public static async Task<string> ReadTextFileAsync(string filepath)
|
||||
{
|
||||
using (var fs = File.Open(filepath, FileMode.Open))
|
||||
{
|
||||
using (var sr = new StreamReader(fs))
|
||||
{
|
||||
return await sr.ReadToEndAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Write data to file.
|
||||
/// </summary>
|
||||
@ -118,6 +149,22 @@ namespace Aki.Common.Utils
|
||||
File.WriteAllBytes(filepath, data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Write data to file.
|
||||
/// </summary>
|
||||
public static async Task WriteFileAsync(string filepath, byte[] data)
|
||||
{
|
||||
if (!Exists(filepath))
|
||||
{
|
||||
CreateDirectory(filepath.GetDirectory());
|
||||
}
|
||||
|
||||
using (FileStream stream = File.Open(filepath, FileMode.OpenOrCreate, FileAccess.Write))
|
||||
{
|
||||
await stream.WriteAsync(data, 0, data.Length);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Write string to file.
|
||||
/// </summary>
|
||||
|
||||
@ -21,7 +21,6 @@ namespace Aki.Custom
|
||||
try
|
||||
{
|
||||
// Bundle patches should always load first
|
||||
BundleManager.GetBundles();
|
||||
new EasyAssetsPatch().Enable();
|
||||
new EasyBundlePatch().Enable();
|
||||
|
||||
|
||||
@ -68,6 +68,12 @@ namespace Aki.Custom.Patches
|
||||
? await GetManifestJson(jsonfile)
|
||||
: await GetManifestBundle(filepath);
|
||||
|
||||
// lazy-initialize aki bundles
|
||||
if (BundleManager.Bundles.Keys.Count == 0)
|
||||
{
|
||||
await BundleManager.GetBundles();
|
||||
}
|
||||
|
||||
// create bundles array from obfuscated type
|
||||
var bundleNames = manifest.GetAllAssetBundles()
|
||||
.Union(BundleManager.Bundles.Keys)
|
||||
@ -122,7 +128,7 @@ namespace Aki.Custom.Patches
|
||||
|
||||
private static async Task<CompatibilityAssetBundleManifest> GetManifestJson(string filepath)
|
||||
{
|
||||
var text = VFS.ReadTextFile(filepath);
|
||||
var text = await VFS.ReadTextFileAsync(filepath);
|
||||
|
||||
/* we cannot parse directly as <string, BundleDetails>, because...
|
||||
[Error : Unity Log] JsonSerializationException: Expected string when reading UnityEngine.Hash128 type, got 'StartObject' <>. Path '['assets/content/weapons/animations/simple_animations.bundle'].Hash', line 1, position 176.
|
||||
|
||||
@ -10,7 +10,7 @@ namespace Aki.Custom.Utils
|
||||
{
|
||||
public static class BundleManager
|
||||
{
|
||||
private static ManualLogSource _logger;
|
||||
private static readonly ManualLogSource _logger;
|
||||
public static readonly ConcurrentDictionary<string, BundleItem> Bundles;
|
||||
public static string CachePath;
|
||||
|
||||
@ -28,7 +28,7 @@ namespace Aki.Custom.Utils
|
||||
: CachePath + bundle.FileName;
|
||||
}
|
||||
|
||||
public static void GetBundles()
|
||||
public static async Task GetBundles()
|
||||
{
|
||||
// get bundles
|
||||
var json = RequestHandler.GetJson("/singleplayer/bundles");
|
||||
@ -37,16 +37,16 @@ namespace Aki.Custom.Utils
|
||||
// register bundles
|
||||
var toDownload = new ConcurrentBag<BundleItem>();
|
||||
|
||||
Parallel.ForEach(bundles, (bundle) =>
|
||||
foreach (var bundle in bundles)
|
||||
{
|
||||
Bundles.TryAdd(bundle.FileName, bundle);
|
||||
|
||||
if (ShouldReaquire(bundle))
|
||||
if (await ShouldReaquire(bundle))
|
||||
{
|
||||
// mark for download
|
||||
toDownload.Add(bundle);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (RequestHandler.IsLocal)
|
||||
{
|
||||
@ -58,17 +58,17 @@ namespace Aki.Custom.Utils
|
||||
{
|
||||
// download bundles
|
||||
// NOTE: assumes bundle keys to be unique
|
||||
Parallel.ForEach(toDownload, (bundle) =>
|
||||
foreach (var bundle in toDownload)
|
||||
{
|
||||
// download bundle
|
||||
var filepath = GetBundlePath(bundle);
|
||||
var data = RequestHandler.GetData($"/files/bundle/{bundle.FileName}");
|
||||
VFS.WriteFile(filepath, data);
|
||||
});
|
||||
var data = await RequestHandler.GetDataAsync($"/files/bundle/{bundle.FileName}");
|
||||
await VFS.WriteFileAsync(filepath, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ShouldReaquire(BundleItem bundle)
|
||||
private static async Task<bool> ShouldReaquire(BundleItem bundle)
|
||||
{
|
||||
if (RequestHandler.IsLocal)
|
||||
{
|
||||
@ -82,7 +82,7 @@ namespace Aki.Custom.Utils
|
||||
if (VFS.Exists(filepath))
|
||||
{
|
||||
// calculate hash
|
||||
var data = VFS.ReadFile(filepath);
|
||||
var data = await VFS.ReadFileAsync(filepath);
|
||||
var crc = Crc32.Compute(data);
|
||||
|
||||
if (crc == bundle.Crc)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user