spt/server
spt/server
0
0
Fork 0
mirror of https://github.com/sp-tarkov/server.git synced 2025-02-13 09:50:43 -05:00
Code

Fully make loadAsync asynchronous (#1053)

Browse Source 
This should make every part that uses `loadAsync` asynchronous

The changes I made:
- I ended up creating a new method to make SHA-1 hashes asynchronously,
did up some reading up and found that `crypto.createHash` could
potentially be blocking.
- Ended up doing some slight code cleanup in `ImporterUtil` to make that
helper more readable.
- I changed `deserializeWithCacheCheckAsync` to skip writing files with
an extra parameter as it was blocking, this can now be called manually
with `writeCacheAsync` (Default behavior of this method stays the same)
This commit is contained in:
Jesse 2025-01-09 17:50:36 +01:00 committed by GitHub
parent 7468975f95
commit ab1b5cd30e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 67 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
project/src/utils/DatabaseImporter.ts
View File

@ -87,7 +87,7 @@ export class DatabaseImporter implements OnLoad {
const dataToImport = await this.importerUtil.loadAsync<IDatabaseTables>(
`${filepath}database/`,
this.filepath,
(fileWithPath: string, data: string) => this.onReadValidate(fileWithPath, data),
async (fileWithPath: string, data: string) => await this.onReadValidate(fileWithPath, data),
);
const validation =
@ -99,9 +99,9 @@ export class DatabaseImporter implements OnLoad {
this.databaseServer.setTables(dataToImport);
}
protected onReadValidate(fileWithPath: string, data: string): void {
protected async onReadValidate(fileWithPath: string, data: string): Promise<void> {
// Validate files
if (ProgramStatics.COMPILED && this.hashedFile && !this.validateFile(fileWithPath, data)) {
if (ProgramStatics.COMPILED && this.hashedFile && !(await this.validateFile(fileWithPath, data))) {
this.valid = VaildationResult.FAILED;
}
}
@ -110,7 +110,7 @@ export class DatabaseImporter implements OnLoad {
return "spt-database";
}
protected validateFile(filePathAndName: string, fileData: any): boolean {
protected async validateFile(filePathAndName: string, fileData: any): Promise<boolean> {
try {
const finalPath = filePathAndName.replace(this.filepath, "").replace(".json", "");
let tempObject: any;
@ -122,7 +122,7 @@ export class DatabaseImporter implements OnLoad {
}
}
if (tempObject !== this.hashUtil.generateSha1ForData(fileData)) {
if (tempObject !== (await this.hashUtil.generateSha1ForDataAsync(fileData))) {
this.logger.debug(this.localisationService.getText("validation_error_file", filePathAndName));
return false;
}

15
project/src/utils/HashUtil.ts
View File

@ -1,4 +1,4 @@
import crypto from "node:crypto";
import crypto, { webcrypto } from "node:crypto";
import { TimeUtil } from "@spt/utils/TimeUtil";
import crc32 from "buffer-crc32";
import { mongoid } from "mongoid-js";
@ -40,7 +40,6 @@ export class HashUtil {
public generateCRC32ForFile(filePath: string): number {
return crc32.unsigned(this.fileSystemSync.read(filePath));
}
/**
* Create a hash for the data parameter
* @param algorithm algorithm to use to hash
@ -53,6 +52,18 @@ export class HashUtil {
return hashSum.digest("hex");
}
/** Creates a SHA-1 hash asynchronously, this doesn't end up blocking.
* @param data data to be hashed
* @returns A promise with the hash value
*/
public async generateSha1ForDataAsync(data: crypto.BinaryLike): Promise<string> {
const encoder = new TextEncoder();
const encodedData = encoder.encode(data.toString());
const hashBuffer = await webcrypto.subtle.digest("SHA-1", encodedData);
return [...new Uint8Array(hashBuffer)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
}
public generateAccountId(): number {
const min = 1000000;
const max = 1999999;

37
project/src/utils/ImporterUtil.ts
View File

@ -13,8 +13,8 @@ export class ImporterUtil {
public async loadAsync<T>(
filepath: string,
strippablePath = "",
onReadCallback: (fileWithPath: string, data: string) => void = () => {},
onObjectDeserialized: (fileWithPath: string, object: any) => void = () => {},
onReadCallback: (fileWithPath: string, data: string) => Promise<void> = () => Promise.resolve(),
onObjectDeserialized: (fileWithPath: string, object: any) => Promise<void> = () => Promise.resolve(),
): Promise<T> {
const result = {} as T;
@ -24,9 +24,9 @@ export class ImporterUtil {
const fileProcessingPromises = allFiles.map(async (file) => {
try {
const fileData = await this.fileSystem.read(file);
onReadCallback(file, fileData);
const fileDeserialized = await this.jsonUtil.deserializeWithCacheCheckAsync<any>(fileData, file);
onObjectDeserialized(file, fileDeserialized);
await onReadCallback(file, fileData);
const fileDeserialized = await this.jsonUtil.deserializeWithCacheCheck<any>(fileData, file, false);
await onObjectDeserialized(file, fileDeserialized);
const strippedFilePath = FileSystem.stripExtension(file).replace(filepath, "");
this.placeObject(fileDeserialized, strippedFilePath, result, strippablePath);
} finally {
@ -35,31 +35,26 @@ export class ImporterUtil {
});
await Promise.all(fileProcessingPromises).catch((e) => console.error(e)); // Wait for promises to resolve
await this.jsonUtil.writeCache(); // Execute writing of all of the hashes one single time
return result;
}
protected placeObject<T>(fileDeserialized: any, strippedFilePath: string, result: T, strippablePath: string): void {
const strippedFinalPath = strippedFilePath.replace(strippablePath, "");
let temp = result;
const propertiesToVisit = strippedFinalPath.split("/");
for (let i = 0; i < propertiesToVisit.length; i++) {
const property = propertiesToVisit[i];
if (i === propertiesToVisit.length - 1) {
temp[property] = fileDeserialized;
// Traverse the object structure
let current = result;
for (const [index, property] of propertiesToVisit.entries()) {
// If we're at the last property, set the value
if (index === propertiesToVisit.length - 1) {
current[property] = fileDeserialized;
} else {
if (!temp[property]) {
temp[property] = {};
}
temp = temp[property];
// Ensure the property exists as an object and move deeper
current[property] = current[property] || {};
current = current[property];
}
}
}
}
class VisitNode {
constructor(
public filePath: string,
public fileName: string,
) {}
}

53
project/src/utils/JsonUtil.ts
View File

@ -1,5 +1,5 @@
import type { ILogger } from "@spt/models/spt/utils/ILogger";
import { FileSystemSync } from "@spt/utils/FileSystemSync";
import { FileSystem } from "@spt/utils/FileSystem";
import { HashUtil } from "@spt/utils/HashUtil";
import { parse, stringify } from "json5";
import { jsonc } from "jsonc";
@ -14,7 +14,7 @@ export class JsonUtil {
protected jsonCachePath = "./user/cache/jsonCache.json";
constructor(
@inject("FileSystemSync") protected fileSystemSync: FileSystemSync,
@inject("FileSystem") protected fileSystem: FileSystem,
@inject("HashUtil") protected hashUtil: HashUtil,
@inject("PrimaryLogger") protected logger: ILogger,
) {}
@ -126,25 +126,23 @@ export class JsonUtil {
}
}
public async deserializeWithCacheCheckAsync<T>(jsonString: string, filePath: string): Promise<T | undefined> {
return new Promise((resolve) => {
resolve(this.deserializeWithCacheCheck<T>(jsonString, filePath));
});
}
/**
* Take json from file and convert into object
* Perform valadation on json during process if json file has not been processed before
* @param jsonString String to turn into object
* @param filePath Path to json file being processed
* @returns Object
* @returns A promise that resolves with the object if successful, if not returns undefined
*/
public deserializeWithCacheCheck<T>(jsonString: string, filePath: string): T | undefined {
this.ensureJsonCacheExists(this.jsonCachePath);
this.hydrateJsonCache(this.jsonCachePath);
public async deserializeWithCacheCheck<T>(
jsonString: string,
filePath: string,
writeHashes = true,
): Promise<T | undefined> {
await this.ensureJsonCacheExists(this.jsonCachePath);
await this.hydrateJsonCache(this.jsonCachePath);
// Generate hash of string
const generatedHash = this.hashUtil.generateSha1ForData(jsonString);
const generatedHash = await this.hashUtil.generateSha1ForDataAsync(jsonString);
if (!this.fileHashes) {
throw new Error("Unable to deserialize with Cache, file hashes have not been hydrated yet");
@ -163,7 +161,11 @@ export class JsonUtil {
} else {
// data valid, save hash and call function again
this.fileHashes[filePath] = generatedHash;
this.fileSystemSync.write(this.jsonCachePath, this.serialize(this.fileHashes, true));
if (writeHashes) {
await this.fileSystem.writeJson(this.jsonCachePath, this.fileHashes);
}
savedHash = generatedHash;
}
return data as T;
@ -184,14 +186,25 @@ export class JsonUtil {
}
/**
* Create file if nothing found
* Writes the file hashes to the cache path, to be used manually if writeHashes was set to false on deserializeWithCacheCheck
*/
public async writeCache(): Promise<void> {
if (!this.fileHashes) {
return;
}
await this.fileSystem.writeJson(this.jsonCachePath, this.fileHashes);
}
/**
* Create file if nothing found asynchronously
* @param jsonCachePath path to cache
*/
protected ensureJsonCacheExists(jsonCachePath: string): void {
protected async ensureJsonCacheExists(jsonCachePath: string): Promise<void> {
if (!this.jsonCacheExists) {
if (!this.fileSystemSync.exists(jsonCachePath)) {
if (!(await this.fileSystem.exists(jsonCachePath))) {
// Create empty object at path
this.fileSystemSync.writeJson(jsonCachePath, {});
await this.fileSystem.writeJson(jsonCachePath, {});
}
this.jsonCacheExists = true;
}
@ -201,10 +214,10 @@ export class JsonUtil {
* Read contents of json cache and add to class field
* @param jsonCachePath Path to cache
*/
protected hydrateJsonCache(jsonCachePath: string): void {
protected async hydrateJsonCache(jsonCachePath: string): Promise<void> {
// Get all file hashes
if (!this.fileHashes) {
this.fileHashes = this.deserialize(this.fileSystemSync.read(`${jsonCachePath}`));
this.fileHashes = await this.fileSystem.readJson(`${jsonCachePath}`);
}
}