Fully make loadAsync asynchronous (#1053)

This should make every part that uses `loadAsync` asynchronous

The changes I made:
- I ended up creating a new method to make SHA-1 hashes asynchronously,
did up some reading up and found that `crypto.createHash` could
potentially be blocking.
- Ended up doing some slight code cleanup in `ImporterUtil` to make that
helper more readable.
- I changed `deserializeWithCacheCheckAsync` to skip writing files with
an extra parameter as it was blocking, this can now be called manually
with `writeCacheAsync` (Default behavior of this method stays the same)

project/src/utils/DatabaseImporter.ts

@@ -87,7 +87,7 @@ export class DatabaseImporter implements OnLoad {
         const dataToImport = await this.importerUtil.loadAsync<IDatabaseTables>(
             `${filepath}database/`,
             this.filepath,
-            (fileWithPath: string, data: string) => this.onReadValidate(fileWithPath, data),
+            async (fileWithPath: string, data: string) => await this.onReadValidate(fileWithPath, data),
         );
 
         const validation =
@@ -99,9 +99,9 @@ export class DatabaseImporter implements OnLoad {
         this.databaseServer.setTables(dataToImport);
     }
 
-    protected onReadValidate(fileWithPath: string, data: string): void {
+    protected async onReadValidate(fileWithPath: string, data: string): Promise<void> {
         // Validate files
-        if (ProgramStatics.COMPILED && this.hashedFile && !this.validateFile(fileWithPath, data)) {
+        if (ProgramStatics.COMPILED && this.hashedFile && !(await this.validateFile(fileWithPath, data))) {
             this.valid = VaildationResult.FAILED;
         }
     }
@@ -110,7 +110,7 @@ export class DatabaseImporter implements OnLoad {
         return "spt-database";
     }
 
-    protected validateFile(filePathAndName: string, fileData: any): boolean {
+    protected async validateFile(filePathAndName: string, fileData: any): Promise<boolean> {
         try {
             const finalPath = filePathAndName.replace(this.filepath, "").replace(".json", "");
             let tempObject: any;
@@ -122,7 +122,7 @@ export class DatabaseImporter implements OnLoad {
                 }
             }
 
-            if (tempObject !== this.hashUtil.generateSha1ForData(fileData)) {
+            if (tempObject !== (await this.hashUtil.generateSha1ForDataAsync(fileData))) {
                 this.logger.debug(this.localisationService.getText("validation_error_file", filePathAndName));
                 return false;
             }

project/src/utils/HashUtil.ts

@@ -1,4 +1,4 @@
-import crypto from "node:crypto";
+import crypto, { webcrypto } from "node:crypto";
 import { TimeUtil } from "@spt/utils/TimeUtil";
 import crc32 from "buffer-crc32";
 import { mongoid } from "mongoid-js";
@@ -40,7 +40,6 @@ export class HashUtil {
     public generateCRC32ForFile(filePath: string): number {
         return crc32.unsigned(this.fileSystemSync.read(filePath));
     }
-
     /**
      * Create a hash for the data parameter
      * @param algorithm algorithm to use to hash
@@ -53,6 +52,18 @@ export class HashUtil {
         return hashSum.digest("hex");
     }
 
+    /** Creates a SHA-1 hash asynchronously, this doesn't end up blocking.
+     * @param data data to be hashed
+     * @returns A promise with the hash value
+     */
+    public async generateSha1ForDataAsync(data: crypto.BinaryLike): Promise<string> {
+        const encoder = new TextEncoder();
+        const encodedData = encoder.encode(data.toString());
+
+        const hashBuffer = await webcrypto.subtle.digest("SHA-1", encodedData);
+        return [...new Uint8Array(hashBuffer)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+    }
+
     public generateAccountId(): number {
         const min = 1000000;
         const max = 1999999;

project/src/utils/ImporterUtil.ts

@@ -13,8 +13,8 @@ export class ImporterUtil {
     public async loadAsync<T>(
         filepath: string,
         strippablePath = "",
-        onReadCallback: (fileWithPath: string, data: string) => void = () => {},
+        onReadCallback: (fileWithPath: string, data: string) => Promise<void> = () => Promise.resolve(),
-        onObjectDeserialized: (fileWithPath: string, object: any) => void = () => {},
+        onObjectDeserialized: (fileWithPath: string, object: any) => Promise<void> = () => Promise.resolve(),
     ): Promise<T> {
         const result = {} as T;
 
@@ -24,9 +24,9 @@ export class ImporterUtil {
         const fileProcessingPromises = allFiles.map(async (file) => {
             try {
                 const fileData = await this.fileSystem.read(file);
-                onReadCallback(file, fileData);
+                await onReadCallback(file, fileData);
-                const fileDeserialized = await this.jsonUtil.deserializeWithCacheCheckAsync<any>(fileData, file);
+                const fileDeserialized = await this.jsonUtil.deserializeWithCacheCheck<any>(fileData, file, false);
-                onObjectDeserialized(file, fileDeserialized);
+                await onObjectDeserialized(file, fileDeserialized);
                 const strippedFilePath = FileSystem.stripExtension(file).replace(filepath, "");
                 this.placeObject(fileDeserialized, strippedFilePath, result, strippablePath);
             } finally {
@@ -35,31 +35,26 @@ export class ImporterUtil {
         });
 
         await Promise.all(fileProcessingPromises).catch((e) => console.error(e)); // Wait for promises to resolve
+        await this.jsonUtil.writeCache(); // Execute writing of all of the hashes one single time
         return result;
     }
 
     protected placeObject<T>(fileDeserialized: any, strippedFilePath: string, result: T, strippablePath: string): void {
         const strippedFinalPath = strippedFilePath.replace(strippablePath, "");
-        let temp = result;
         const propertiesToVisit = strippedFinalPath.split("/");
-        for (let i = 0; i < propertiesToVisit.length; i++) {
-            const property = propertiesToVisit[i];
 
-            if (i === propertiesToVisit.length - 1) {
+        // Traverse the object structure
-                temp[property] = fileDeserialized;
+        let current = result;
+
+        for (const [index, property] of propertiesToVisit.entries()) {
+            // If we're at the last property, set the value
+            if (index === propertiesToVisit.length - 1) {
+                current[property] = fileDeserialized;
             } else {
-                if (!temp[property]) {
+                // Ensure the property exists as an object and move deeper
-                    temp[property] = {};
+                current[property] = current[property] || {};
-                }
+                current = current[property];
-                temp = temp[property];
             }
         }
     }
 }
-
-class VisitNode {
-    constructor(
-        public filePath: string,
-        public fileName: string,
-    ) {}
-}

project/src/utils/JsonUtil.ts

@@ -1,5 +1,5 @@
 import type { ILogger } from "@spt/models/spt/utils/ILogger";
-import { FileSystemSync } from "@spt/utils/FileSystemSync";
+import { FileSystem } from "@spt/utils/FileSystem";
 import { HashUtil } from "@spt/utils/HashUtil";
 import { parse, stringify } from "json5";
 import { jsonc } from "jsonc";
@@ -14,7 +14,7 @@ export class JsonUtil {
     protected jsonCachePath = "./user/cache/jsonCache.json";
 
     constructor(
-        @inject("FileSystemSync") protected fileSystemSync: FileSystemSync,
+        @inject("FileSystem") protected fileSystem: FileSystem,
         @inject("HashUtil") protected hashUtil: HashUtil,
         @inject("PrimaryLogger") protected logger: ILogger,
     ) {}
@@ -126,25 +126,23 @@ export class JsonUtil {
         }
     }
 
-    public async deserializeWithCacheCheckAsync<T>(jsonString: string, filePath: string): Promise<T | undefined> {
-        return new Promise((resolve) => {
-            resolve(this.deserializeWithCacheCheck<T>(jsonString, filePath));
-        });
-    }
-
     /**
      * Take json from file and convert into object
      * Perform valadation on json during process if json file has not been processed before
      * @param jsonString String to turn into object
      * @param filePath Path to json file being processed
-     * @returns Object
+     * @returns A promise that resolves with the object if successful, if not returns undefined
      */
-    public deserializeWithCacheCheck<T>(jsonString: string, filePath: string): T | undefined {
+    public async deserializeWithCacheCheck<T>(
-        this.ensureJsonCacheExists(this.jsonCachePath);
+        jsonString: string,
-        this.hydrateJsonCache(this.jsonCachePath);
+        filePath: string,
+        writeHashes = true,
+    ): Promise<T | undefined> {
+        await this.ensureJsonCacheExists(this.jsonCachePath);
+        await this.hydrateJsonCache(this.jsonCachePath);
 
         // Generate hash of string
-        const generatedHash = this.hashUtil.generateSha1ForData(jsonString);
+        const generatedHash = await this.hashUtil.generateSha1ForDataAsync(jsonString);
 
         if (!this.fileHashes) {
             throw new Error("Unable to deserialize with Cache, file hashes have not been hydrated yet");
@@ -163,7 +161,11 @@ export class JsonUtil {
                 } else {
                     // data valid, save hash and call function again
                     this.fileHashes[filePath] = generatedHash;
-                    this.fileSystemSync.write(this.jsonCachePath, this.serialize(this.fileHashes, true));
+
+                    if (writeHashes) {
+                        await this.fileSystem.writeJson(this.jsonCachePath, this.fileHashes);
+                    }
+
                     savedHash = generatedHash;
                 }
                 return data as T;
@@ -184,14 +186,25 @@ export class JsonUtil {
     }
 
     /**
-     * Create file if nothing found
+     * Writes the file hashes to the cache path, to be used manually if writeHashes was set to false on deserializeWithCacheCheck
+     */
+    public async writeCache(): Promise<void> {
+        if (!this.fileHashes) {
+            return;
+        }
+
+        await this.fileSystem.writeJson(this.jsonCachePath, this.fileHashes);
+    }
+
+    /**
+     * Create file if nothing found asynchronously
      * @param jsonCachePath path to cache
      */
-    protected ensureJsonCacheExists(jsonCachePath: string): void {
+    protected async ensureJsonCacheExists(jsonCachePath: string): Promise<void> {
         if (!this.jsonCacheExists) {
-            if (!this.fileSystemSync.exists(jsonCachePath)) {
+            if (!(await this.fileSystem.exists(jsonCachePath))) {
                 // Create empty object at path
-                this.fileSystemSync.writeJson(jsonCachePath, {});
+                await this.fileSystem.writeJson(jsonCachePath, {});
             }
             this.jsonCacheExists = true;
         }
@@ -201,10 +214,10 @@ export class JsonUtil {
      * Read contents of json cache and add to class field
      * @param jsonCachePath Path to cache
      */
-    protected hydrateJsonCache(jsonCachePath: string): void {
+    protected async hydrateJsonCache(jsonCachePath: string): Promise<void> {
         // Get all file hashes
         if (!this.fileHashes) {
-            this.fileHashes = this.deserialize(this.fileSystemSync.read(`${jsonCachePath}`));
+            this.fileHashes = await this.fileSystem.readJson(`${jsonCachePath}`);
         }
     }