From ab1b5cd30ef0db6f05ee09d8b51dcff35120f18d Mon Sep 17 00:00:00 2001 From: Jesse Date: Thu, 9 Jan 2025 17:50:36 +0100 Subject: [PATCH] Fully make loadAsync asynchronous (#1053) This should make every part that uses `loadAsync` asynchronous The changes I made: - I ended up creating a new method to make SHA-1 hashes asynchronously, did up some reading up and found that `crypto.createHash` could potentially be blocking. - Ended up doing some slight code cleanup in `ImporterUtil` to make that helper more readable. - I changed `deserializeWithCacheCheckAsync` to skip writing files with an extra parameter as it was blocking, this can now be called manually with `writeCacheAsync` (Default behavior of this method stays the same) --- project/src/utils/DatabaseImporter.ts | 10 ++--- project/src/utils/HashUtil.ts | 15 +++++++- project/src/utils/ImporterUtil.ts | 37 ++++++++----------- project/src/utils/JsonUtil.ts | 53 +++++++++++++++++---------- 4 files changed, 67 insertions(+), 48 deletions(-) diff --git a/project/src/utils/DatabaseImporter.ts b/project/src/utils/DatabaseImporter.ts index 7bddfc0f..587cc974 100644 --- a/project/src/utils/DatabaseImporter.ts +++ b/project/src/utils/DatabaseImporter.ts @@ -87,7 +87,7 @@ export class DatabaseImporter implements OnLoad { const dataToImport = await this.importerUtil.loadAsync( `${filepath}database/`, this.filepath, - (fileWithPath: string, data: string) => this.onReadValidate(fileWithPath, data), + async (fileWithPath: string, data: string) => await this.onReadValidate(fileWithPath, data), ); const validation = @@ -99,9 +99,9 @@ export class DatabaseImporter implements OnLoad { this.databaseServer.setTables(dataToImport); } - protected onReadValidate(fileWithPath: string, data: string): void { + protected async onReadValidate(fileWithPath: string, data: string): Promise { // Validate files - if (ProgramStatics.COMPILED && this.hashedFile && !this.validateFile(fileWithPath, data)) { + if (ProgramStatics.COMPILED && this.hashedFile && !(await this.validateFile(fileWithPath, data))) { this.valid = VaildationResult.FAILED; } } @@ -110,7 +110,7 @@ export class DatabaseImporter implements OnLoad { return "spt-database"; } - protected validateFile(filePathAndName: string, fileData: any): boolean { + protected async validateFile(filePathAndName: string, fileData: any): Promise { try { const finalPath = filePathAndName.replace(this.filepath, "").replace(".json", ""); let tempObject: any; @@ -122,7 +122,7 @@ export class DatabaseImporter implements OnLoad { } } - if (tempObject !== this.hashUtil.generateSha1ForData(fileData)) { + if (tempObject !== (await this.hashUtil.generateSha1ForDataAsync(fileData))) { this.logger.debug(this.localisationService.getText("validation_error_file", filePathAndName)); return false; } diff --git a/project/src/utils/HashUtil.ts b/project/src/utils/HashUtil.ts index aaac84d4..a9e3deb0 100644 --- a/project/src/utils/HashUtil.ts +++ b/project/src/utils/HashUtil.ts @@ -1,4 +1,4 @@ -import crypto from "node:crypto"; +import crypto, { webcrypto } from "node:crypto"; import { TimeUtil } from "@spt/utils/TimeUtil"; import crc32 from "buffer-crc32"; import { mongoid } from "mongoid-js"; @@ -40,7 +40,6 @@ export class HashUtil { public generateCRC32ForFile(filePath: string): number { return crc32.unsigned(this.fileSystemSync.read(filePath)); } - /** * Create a hash for the data parameter * @param algorithm algorithm to use to hash @@ -53,6 +52,18 @@ export class HashUtil { return hashSum.digest("hex"); } + /** Creates a SHA-1 hash asynchronously, this doesn't end up blocking. + * @param data data to be hashed + * @returns A promise with the hash value + */ + public async generateSha1ForDataAsync(data: crypto.BinaryLike): Promise { + const encoder = new TextEncoder(); + const encodedData = encoder.encode(data.toString()); + + const hashBuffer = await webcrypto.subtle.digest("SHA-1", encodedData); + return [...new Uint8Array(hashBuffer)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); + } + public generateAccountId(): number { const min = 1000000; const max = 1999999; diff --git a/project/src/utils/ImporterUtil.ts b/project/src/utils/ImporterUtil.ts index 3360c82d..11d5b5c1 100644 --- a/project/src/utils/ImporterUtil.ts +++ b/project/src/utils/ImporterUtil.ts @@ -13,8 +13,8 @@ export class ImporterUtil { public async loadAsync( filepath: string, strippablePath = "", - onReadCallback: (fileWithPath: string, data: string) => void = () => {}, - onObjectDeserialized: (fileWithPath: string, object: any) => void = () => {}, + onReadCallback: (fileWithPath: string, data: string) => Promise = () => Promise.resolve(), + onObjectDeserialized: (fileWithPath: string, object: any) => Promise = () => Promise.resolve(), ): Promise { const result = {} as T; @@ -24,9 +24,9 @@ export class ImporterUtil { const fileProcessingPromises = allFiles.map(async (file) => { try { const fileData = await this.fileSystem.read(file); - onReadCallback(file, fileData); - const fileDeserialized = await this.jsonUtil.deserializeWithCacheCheckAsync(fileData, file); - onObjectDeserialized(file, fileDeserialized); + await onReadCallback(file, fileData); + const fileDeserialized = await this.jsonUtil.deserializeWithCacheCheck(fileData, file, false); + await onObjectDeserialized(file, fileDeserialized); const strippedFilePath = FileSystem.stripExtension(file).replace(filepath, ""); this.placeObject(fileDeserialized, strippedFilePath, result, strippablePath); } finally { @@ -35,31 +35,26 @@ export class ImporterUtil { }); await Promise.all(fileProcessingPromises).catch((e) => console.error(e)); // Wait for promises to resolve + await this.jsonUtil.writeCache(); // Execute writing of all of the hashes one single time return result; } protected placeObject(fileDeserialized: any, strippedFilePath: string, result: T, strippablePath: string): void { const strippedFinalPath = strippedFilePath.replace(strippablePath, ""); - let temp = result; const propertiesToVisit = strippedFinalPath.split("/"); - for (let i = 0; i < propertiesToVisit.length; i++) { - const property = propertiesToVisit[i]; - if (i === propertiesToVisit.length - 1) { - temp[property] = fileDeserialized; + // Traverse the object structure + let current = result; + + for (const [index, property] of propertiesToVisit.entries()) { + // If we're at the last property, set the value + if (index === propertiesToVisit.length - 1) { + current[property] = fileDeserialized; } else { - if (!temp[property]) { - temp[property] = {}; - } - temp = temp[property]; + // Ensure the property exists as an object and move deeper + current[property] = current[property] || {}; + current = current[property]; } } } } - -class VisitNode { - constructor( - public filePath: string, - public fileName: string, - ) {} -} diff --git a/project/src/utils/JsonUtil.ts b/project/src/utils/JsonUtil.ts index ad71a7fc..be32c095 100644 --- a/project/src/utils/JsonUtil.ts +++ b/project/src/utils/JsonUtil.ts @@ -1,5 +1,5 @@ import type { ILogger } from "@spt/models/spt/utils/ILogger"; -import { FileSystemSync } from "@spt/utils/FileSystemSync"; +import { FileSystem } from "@spt/utils/FileSystem"; import { HashUtil } from "@spt/utils/HashUtil"; import { parse, stringify } from "json5"; import { jsonc } from "jsonc"; @@ -14,7 +14,7 @@ export class JsonUtil { protected jsonCachePath = "./user/cache/jsonCache.json"; constructor( - @inject("FileSystemSync") protected fileSystemSync: FileSystemSync, + @inject("FileSystem") protected fileSystem: FileSystem, @inject("HashUtil") protected hashUtil: HashUtil, @inject("PrimaryLogger") protected logger: ILogger, ) {} @@ -126,25 +126,23 @@ export class JsonUtil { } } - public async deserializeWithCacheCheckAsync(jsonString: string, filePath: string): Promise { - return new Promise((resolve) => { - resolve(this.deserializeWithCacheCheck(jsonString, filePath)); - }); - } - /** * Take json from file and convert into object * Perform valadation on json during process if json file has not been processed before * @param jsonString String to turn into object * @param filePath Path to json file being processed - * @returns Object + * @returns A promise that resolves with the object if successful, if not returns undefined */ - public deserializeWithCacheCheck(jsonString: string, filePath: string): T | undefined { - this.ensureJsonCacheExists(this.jsonCachePath); - this.hydrateJsonCache(this.jsonCachePath); + public async deserializeWithCacheCheck( + jsonString: string, + filePath: string, + writeHashes = true, + ): Promise { + await this.ensureJsonCacheExists(this.jsonCachePath); + await this.hydrateJsonCache(this.jsonCachePath); // Generate hash of string - const generatedHash = this.hashUtil.generateSha1ForData(jsonString); + const generatedHash = await this.hashUtil.generateSha1ForDataAsync(jsonString); if (!this.fileHashes) { throw new Error("Unable to deserialize with Cache, file hashes have not been hydrated yet"); @@ -163,7 +161,11 @@ export class JsonUtil { } else { // data valid, save hash and call function again this.fileHashes[filePath] = generatedHash; - this.fileSystemSync.write(this.jsonCachePath, this.serialize(this.fileHashes, true)); + + if (writeHashes) { + await this.fileSystem.writeJson(this.jsonCachePath, this.fileHashes); + } + savedHash = generatedHash; } return data as T; @@ -184,14 +186,25 @@ export class JsonUtil { } /** - * Create file if nothing found + * Writes the file hashes to the cache path, to be used manually if writeHashes was set to false on deserializeWithCacheCheck + */ + public async writeCache(): Promise { + if (!this.fileHashes) { + return; + } + + await this.fileSystem.writeJson(this.jsonCachePath, this.fileHashes); + } + + /** + * Create file if nothing found asynchronously * @param jsonCachePath path to cache */ - protected ensureJsonCacheExists(jsonCachePath: string): void { + protected async ensureJsonCacheExists(jsonCachePath: string): Promise { if (!this.jsonCacheExists) { - if (!this.fileSystemSync.exists(jsonCachePath)) { + if (!(await this.fileSystem.exists(jsonCachePath))) { // Create empty object at path - this.fileSystemSync.writeJson(jsonCachePath, {}); + await this.fileSystem.writeJson(jsonCachePath, {}); } this.jsonCacheExists = true; } @@ -201,10 +214,10 @@ export class JsonUtil { * Read contents of json cache and add to class field * @param jsonCachePath Path to cache */ - protected hydrateJsonCache(jsonCachePath: string): void { + protected async hydrateJsonCache(jsonCachePath: string): Promise { // Get all file hashes if (!this.fileHashes) { - this.fileHashes = this.deserialize(this.fileSystemSync.read(`${jsonCachePath}`)); + this.fileHashes = await this.fileSystem.readJson(`${jsonCachePath}`); } }