diff --git a/project/package.json b/project/package.json index b51f4f41..eaeae684 100644 --- a/project/package.json +++ b/project/package.json @@ -32,6 +32,7 @@ }, "dependencies": { "atomically": "~1.7", + "buffer-crc32": "^1.0.0", "date-fns": "~2.30", "date-fns-tz": "~2.0", "i18n": "~0.15", diff --git a/project/src/callbacks/BundleCallbacks.ts b/project/src/callbacks/BundleCallbacks.ts index 28317854..260159d2 100644 --- a/project/src/callbacks/BundleCallbacks.ts +++ b/project/src/callbacks/BundleCallbacks.ts @@ -25,8 +25,7 @@ export class BundleCallbacks */ public getBundles(url: string, info: any, sessionID: string): string { - const local = this.httpConfig.ip === "127.0.0.1" || this.httpConfig.ip === "localhost"; - return this.httpResponse.noBody(this.bundleLoader.getBundles(local)); + return this.httpResponse.noBody(this.bundleLoader.getBundles()); } public getBundle(url: string, info: any, sessionID: string): string diff --git a/project/src/di/Container.ts b/project/src/di/Container.ts index 89f33a70..30ab1fb1 100644 --- a/project/src/di/Container.ts +++ b/project/src/di/Container.ts @@ -199,7 +199,6 @@ import { BotWeaponModLimitService } from "@spt-aki/services/BotWeaponModLimitSer import { CustomLocationWaveService } from "@spt-aki/services/CustomLocationWaveService"; import { FenceService } from "@spt-aki/services/FenceService"; import { GiftService } from "@spt-aki/services/GiftService"; -import { HashCacheService } from "@spt-aki/services/HashCacheService"; import { InsuranceService } from "@spt-aki/services/InsuranceService"; import { ItemBaseClassService } from "@spt-aki/services/ItemBaseClassService"; import { ItemFilterService } from "@spt-aki/services/ItemFilterService"; @@ -228,6 +227,8 @@ import { SeasonalEventService } from "@spt-aki/services/SeasonalEventService"; import { TraderAssortService } from "@spt-aki/services/TraderAssortService"; import { TraderPurchasePersisterService } from "@spt-aki/services/TraderPurchasePersisterService"; import { TraderServicesService } from "@spt-aki/services/TraderServicesService"; +import { BundleHashCacheService } from "@spt-aki/services/cache/BundleHashCacheService"; +import { ModHashCacheService } from "@spt-aki/services/cache/ModHashCacheService"; import { CustomItemService } from "@spt-aki/services/mod/CustomItemService"; import { DynamicRouterModService } from "@spt-aki/services/mod/dynamicRouter/DynamicRouterModService"; import { HttpListenerModService } from "@spt-aki/services/mod/httpListener/HttpListenerModService"; @@ -690,7 +691,10 @@ export class Container lifecycle: Lifecycle.Singleton, }); depContainer.register("ModCompilerService", ModCompilerService); - depContainer.register("HashCacheService", HashCacheService, { + depContainer.register("BundleHashCacheService", BundleHashCacheService, { + lifecycle: Lifecycle.Singleton, + }); + depContainer.register("ModHashCacheService", ModHashCacheService, { lifecycle: Lifecycle.Singleton, }); depContainer.register("LocaleService", LocaleService, { lifecycle: Lifecycle.Singleton }); diff --git a/project/src/loaders/BundleLoader.ts b/project/src/loaders/BundleLoader.ts index 1a7be5a2..657fd970 100644 --- a/project/src/loaders/BundleLoader.ts +++ b/project/src/loaders/BundleLoader.ts @@ -1,25 +1,24 @@ +import path from "node:path"; import { inject, injectable } from "tsyringe"; -import path from "path"; import { HttpServerHelper } from "@spt-aki/helpers/HttpServerHelper"; +import { BundleHashCacheService } from "@spt-aki/services/cache/BundleHashCacheService"; import { JsonUtil } from "@spt-aki/utils/JsonUtil"; import { VFS } from "@spt-aki/utils/VFS"; export class BundleInfo { - modPath: string; - key: string; - path: string; - filepath: string; - dependencyKeys: string[]; + modpath: string; + filename: string; + crc: number; + dependencies: string[]; - constructor(modpath: string, bundle: any, bundlePath: string, bundleFilepath: string) + constructor(modpath: string, bundle: BundleManifestEntry, bundleHash: number) { - this.modPath = modpath; - this.key = bundle.key; - this.path = bundlePath; - this.filepath = bundleFilepath; - this.dependencyKeys = bundle.dependencyKeys || []; + this.modpath = modpath; + this.filename = bundle.key; + this.crc = bundleHash; + this.dependencies = bundle.dependencyKeys || []; } } @@ -32,47 +31,48 @@ export class BundleLoader @inject("HttpServerHelper") protected httpServerHelper: HttpServerHelper, @inject("VFS") protected vfs: VFS, @inject("JsonUtil") protected jsonUtil: JsonUtil, + @inject("BundleHashCacheService") protected bundleHashCacheService: BundleHashCacheService, ) {} /** * Handle singleplayer/bundles */ - public getBundles(local: boolean): BundleInfo[] + public getBundles(): BundleInfo[] { const result: BundleInfo[] = []; for (const bundle in this.bundles) { - result.push(this.getBundle(bundle, local)); + result.push(this.getBundle(bundle)); } return result; } - public getBundle(key: string, local: boolean): BundleInfo + public getBundle(key: string): BundleInfo { - const bundle = this.jsonUtil.clone(this.bundles[key]); - - if (local) - { - bundle.path = path.join(process.cwd(), bundle.filepath); - } - - delete bundle.filepath; - return bundle; + return this.jsonUtil.clone(this.bundles[key]); } public addBundles(modpath: string): void { - const manifest = + const bundleManifestArr = this.jsonUtil.deserialize(this.vfs.readFile(`${modpath}bundles.json`)).manifest; - for (const bundle of manifest) + for (const bundleManifest of bundleManifestArr) { - const bundlePath = `${this.httpServerHelper.getBackendUrl()}/files/bundle/${bundle.key}`; - const bundleFilepath = bundle.path || `${modpath}bundles/${bundle.key}`.replace(/\\/g, "/"); - this.addBundle(bundle.key, new BundleInfo(modpath, bundle, bundlePath, bundleFilepath)); + const absoluteModPath = path.join(process.cwd(), modpath).slice(0, -1).replace(/\\/g, "/"); + const bundleLocalPath = `${modpath}bundles/${bundleManifest.key}`.replace(/\\/g, "/"); + + if (!this.bundleHashCacheService.calculateAndMatchHash(bundleLocalPath)) + { + this.bundleHashCacheService.calculateAndStoreHash(bundleLocalPath); + } + + const bundleHash = this.bundleHashCacheService.getStoredValue(bundleLocalPath); + + this.addBundle(bundleManifest.key, new BundleInfo(absoluteModPath, bundleManifest, bundleHash)); } } @@ -84,11 +84,11 @@ export class BundleLoader export interface BundleManifest { - manifest: Array; + manifest: BundleManifestEntry[]; } export interface BundleManifestEntry { key: string; - path: string; + dependencyKeys: string[]; } diff --git a/project/src/routers/serializers/BundleSerializer.ts b/project/src/routers/serializers/BundleSerializer.ts index 691f7dee..df8c8866 100644 --- a/project/src/routers/serializers/BundleSerializer.ts +++ b/project/src/routers/serializers/BundleSerializer.ts @@ -23,10 +23,9 @@ export class BundleSerializer extends Serializer this.logger.info(`[BUNDLE]: ${req.url}`); const key = req.url.split("/bundle/")[1]; - const bundle = this.bundleLoader.getBundle(key, true); + const bundle = this.bundleLoader.getBundle(key); - // send bundle - this.httpFileUtil.sendFile(resp, bundle.path); + this.httpFileUtil.sendFile(resp, `${bundle.modpath}/bundles/${bundle.filename}`); } public override canHandle(route: string): boolean diff --git a/project/src/servers/HttpServer.ts b/project/src/servers/HttpServer.ts index d038edd6..421c7ae4 100644 --- a/project/src/servers/HttpServer.ts +++ b/project/src/servers/HttpServer.ts @@ -1,4 +1,4 @@ -import http, { IncomingMessage, ServerResponse } from "node:http"; +import http, { IncomingMessage, ServerResponse, Server } from "node:http"; import { inject, injectAll, injectable } from "tsyringe"; import { ApplicationContext } from "@spt-aki/context/ApplicationContext"; @@ -38,7 +38,9 @@ export class HttpServer public load(): void { /* create server */ - const httpServer: http.Server = http.createServer((req, res) => + const httpServer: Server = http.createServer(); + + httpServer.on("request", (req, res) => { this.handleRequest(req, res); }); @@ -104,7 +106,7 @@ export class HttpServer } } - protected getCookies(req: http.IncomingMessage): Record + protected getCookies(req: IncomingMessage): Record { const found: Record = {}; const cookies = req.headers.cookie; diff --git a/project/src/servers/http/AkiHttpListener.ts b/project/src/servers/http/AkiHttpListener.ts index 07ec48f6..34913e49 100644 --- a/project/src/servers/http/AkiHttpListener.ts +++ b/project/src/servers/http/AkiHttpListener.ts @@ -48,7 +48,7 @@ export class AkiHttpListener implements IHttpListener // kinda big), on a slow connection. We need to re-assemble the entire http payload // before processing it. - const requestLength = parseInt(req.headers["content-length"]); + const requestLength = Number.parseInt(req.headers["content-length"]); const buffer = Buffer.alloc(requestLength); let written = 0; diff --git a/project/src/services/HashCacheService.ts b/project/src/services/HashCacheService.ts deleted file mode 100644 index 6b98f7ec..00000000 --- a/project/src/services/HashCacheService.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { inject, injectable } from "tsyringe"; - -import { ILogger } from "@spt-aki/models/spt/utils/ILogger"; -import { HashUtil } from "@spt-aki/utils/HashUtil"; -import { JsonUtil } from "@spt-aki/utils/JsonUtil"; -import { VFS } from "@spt-aki/utils/VFS"; - -@injectable() -export class HashCacheService -{ - protected jsonHashes = null; - protected modHashes = null; - protected readonly modCachePath = "./user/cache/modCache.json"; - - constructor( - @inject("VFS") protected vfs: VFS, - @inject("HashUtil") protected hashUtil: HashUtil, - @inject("JsonUtil") protected jsonUtil: JsonUtil, - @inject("WinstonLogger") protected logger: ILogger, - ) - { - if (!this.vfs.exists(this.modCachePath)) - { - this.vfs.writeFile(this.modCachePath, "{}"); - } - - // get mod hash file - if (!this.modHashes) - { // empty - this.modHashes = this.jsonUtil.deserialize(this.vfs.readFile(`${this.modCachePath}`), this.modCachePath); - } - } - - /** - * Return a stored hash by key - * @param modName Name of mod to get hash for - * @returns Mod hash - */ - public getStoredModHash(modName: string): string - { - return this.modHashes[modName]; - } - - /** - * Does the generated hash match the stored hash - * @param modName name of mod - * @param modContent - * @returns True if they match - */ - public modContentMatchesStoredHash(modName: string, modContent: string): boolean - { - const storedModHash = this.getStoredModHash(modName); - const generatedHash = this.hashUtil.generateSha1ForData(modContent); - - return storedModHash === generatedHash; - } - - public hashMatchesStoredHash(modName: string, modHash: string): boolean - { - const storedModHash = this.getStoredModHash(modName); - - return storedModHash === modHash; - } - - public storeModContent(modName: string, modContent: string): void - { - const generatedHash = this.hashUtil.generateSha1ForData(modContent); - - this.storeModHash(modName, generatedHash); - } - - public storeModHash(modName: string, modHash: string): void - { - this.modHashes[modName] = modHash; - - this.vfs.writeFile(this.modCachePath, this.jsonUtil.serialize(this.modHashes)); - - this.logger.debug(`Mod ${modName} hash stored in ${this.modCachePath}`); - } -} diff --git a/project/src/services/ModCompilerService.ts b/project/src/services/ModCompilerService.ts index 833c22d7..159588b9 100644 --- a/project/src/services/ModCompilerService.ts +++ b/project/src/services/ModCompilerService.ts @@ -5,7 +5,7 @@ import { inject, injectable } from "tsyringe"; import ts from "typescript"; import type { ILogger } from "@spt-aki/models/spt/utils/ILogger"; -import { HashCacheService } from "@spt-aki/services/HashCacheService"; +import { ModHashCacheService } from "@spt-aki/services/cache/ModHashCacheService"; import { VFS } from "@spt-aki/utils/VFS"; @injectable() @@ -15,7 +15,7 @@ export class ModCompilerService constructor( @inject("WinstonLogger") protected logger: ILogger, - @inject("HashCacheService") protected hashCacheService: HashCacheService, + @inject("ModHashCacheService") protected modHashCacheService: ModHashCacheService, @inject("VFS") protected vfs: VFS, ) { @@ -47,7 +47,7 @@ export class ModCompilerService } } - const hashMatches = this.hashCacheService.modContentMatchesStoredHash(modName, tsFileContents); + const hashMatches = this.modHashCacheService.calculateAndCompareHash(modName, tsFileContents); if (fileExists && hashMatches) { @@ -58,7 +58,7 @@ export class ModCompilerService if (!hashMatches) { // Store / update hash in json file - this.hashCacheService.storeModContent(modName, tsFileContents); + this.modHashCacheService.calculateAndStoreHash(modName, tsFileContents); } return this.compile(modTypeScriptFiles, { diff --git a/project/src/services/cache/BundleHashCacheService.ts b/project/src/services/cache/BundleHashCacheService.ts new file mode 100644 index 00000000..82730244 --- /dev/null +++ b/project/src/services/cache/BundleHashCacheService.ts @@ -0,0 +1,64 @@ +import { inject, injectable } from "tsyringe"; + +import { ILogger } from "@spt-aki/models/spt/utils/ILogger"; +import { HashUtil } from "@spt-aki/utils/HashUtil"; +import { JsonUtil } from "@spt-aki/utils/JsonUtil"; +import { VFS } from "@spt-aki/utils/VFS"; + +@injectable() +export class BundleHashCacheService +{ + protected bundleHashes: Record; + protected readonly bundleHashCachePath = "./user/cache/bundleHashCache.json"; + + constructor( + @inject("VFS") protected vfs: VFS, + @inject("HashUtil") protected hashUtil: HashUtil, + @inject("JsonUtil") protected jsonUtil: JsonUtil, + @inject("WinstonLogger") protected logger: ILogger, + ) + { + if (!this.vfs.exists(this.bundleHashCachePath)) + { + this.vfs.writeFile(this.bundleHashCachePath, "{}"); + } + + this.bundleHashes = this.jsonUtil.deserialize( + this.vfs.readFile(this.bundleHashCachePath), + this.bundleHashCachePath, + ); + } + + public getStoredValue(key: string): number + { + return this.bundleHashes[key]; + } + + public storeValue(key: string, value: number): void + { + this.bundleHashes[key] = value; + + this.vfs.writeFile(this.bundleHashCachePath, this.jsonUtil.serialize(this.bundleHashes)); + + this.logger.debug(`Bundle ${key} hash stored in ${this.bundleHashCachePath}`); + } + + public matchWithStoredHash(bundlePath: string, hash: number): boolean + { + return this.getStoredValue(bundlePath) === hash; + } + + public calculateAndMatchHash(bundlePath: string): boolean + { + const generatedHash = this.hashUtil.generateCRC32ForFile(bundlePath); + + return this.matchWithStoredHash(bundlePath, generatedHash); + } + + public calculateAndStoreHash(bundlePath: string): void + { + const generatedHash = this.hashUtil.generateCRC32ForFile(bundlePath); + + this.storeValue(bundlePath, generatedHash); + } +} diff --git a/project/src/services/cache/ModHashCacheService.ts b/project/src/services/cache/ModHashCacheService.ts new file mode 100644 index 00000000..1edd6bba --- /dev/null +++ b/project/src/services/cache/ModHashCacheService.ts @@ -0,0 +1,61 @@ +import { inject, injectable } from "tsyringe"; + +import { ILogger } from "@spt-aki/models/spt/utils/ILogger"; +import { HashUtil } from "@spt-aki/utils/HashUtil"; +import { JsonUtil } from "@spt-aki/utils/JsonUtil"; +import { VFS } from "@spt-aki/utils/VFS"; + +@injectable() +export class ModHashCacheService +{ + protected modHashes: Record; + protected readonly modCachePath = "./user/cache/modCache.json"; + + constructor( + @inject("VFS") protected vfs: VFS, + @inject("HashUtil") protected hashUtil: HashUtil, + @inject("JsonUtil") protected jsonUtil: JsonUtil, + @inject("WinstonLogger") protected logger: ILogger, + ) + { + if (!this.vfs.exists(this.modCachePath)) + { + this.vfs.writeFile(this.modCachePath, "{}"); + } + + this.modHashes = this.jsonUtil.deserialize(this.vfs.readFile(this.modCachePath), this.modCachePath); + } + + public getStoredValue(key: string): string + { + return this.modHashes[key]; + } + + public storeValue(key: string, value: string): void + { + this.modHashes[key] = value; + + this.vfs.writeFile(this.modCachePath, this.jsonUtil.serialize(this.modHashes)); + + this.logger.debug(`Mod ${key} hash stored in ${this.modCachePath}`); + } + + public matchWithStoredHash(modName: string, hash: string): boolean + { + return this.getStoredValue(modName) === hash; + } + + public calculateAndCompareHash(modName: string, modContent: string): boolean + { + const generatedHash = this.hashUtil.generateSha1ForData(modContent); + + return this.matchWithStoredHash(modName, generatedHash); + } + + public calculateAndStoreHash(modName: string, modContent: string): void + { + const generatedHash = this.hashUtil.generateSha1ForData(modContent); + + this.storeValue(modName, generatedHash); + } +} diff --git a/project/src/utils/HashUtil.ts b/project/src/utils/HashUtil.ts index 2ef10596..229ac816 100644 --- a/project/src/utils/HashUtil.ts +++ b/project/src/utils/HashUtil.ts @@ -1,4 +1,6 @@ import crypto from "node:crypto"; +import fs from "node:fs"; +import crc32 from "buffer-crc32"; import { inject, injectable } from "tsyringe"; import { TimeUtil } from "@spt-aki/utils/TimeUtil"; @@ -32,6 +34,11 @@ export class HashUtil return this.generateHashForData("sha1", data); } + public generateCRC32ForFile(filePath: fs.PathLike): number + { + return crc32.unsigned(fs.readFileSync(filePath)); + } + /** * Create a hash for the data parameter * @param algorithm algorithm to use to hash diff --git a/project/src/utils/HttpFileUtil.ts b/project/src/utils/HttpFileUtil.ts index 37f1bb35..621aa134 100644 --- a/project/src/utils/HttpFileUtil.ts +++ b/project/src/utils/HttpFileUtil.ts @@ -11,12 +11,12 @@ export class HttpFileUtil { } - public sendFile(resp: ServerResponse, file: any): void + public sendFile(resp: ServerResponse, filePath: string): void { - const pathSlic = file.split("/"); + const pathSlic = filePath.split("/"); const type = this.httpServerHelper.getMimeText(pathSlic[pathSlic.length - 1].split(".").at(-1)) || this.httpServerHelper.getMimeText("txt"); - const fileStream = fs.createReadStream(file); + const fileStream = fs.createReadStream(filePath); fileStream.on("open", () => {