Add JSONC support to server configs + use by modders (!112)

Replaced calls (where possible) to JSON.parse/stringify with use of `jsonUtil` functions
`VFS.ts` was tricky, it can't be updated as it'd create a circular dependency

Also add json5 to package.json for modders to have access to

Co-authored-by: Dev <dev@dev.sp-tarkov.com>
Reviewed-on: SPT-AKI/Server#112

project/gulpfile.mjs

@@ -80,7 +80,7 @@ const packagingBleeding = async () => pkg.exec([entries.bleeding, "--compression
 // Assets
 const addAssets = async (cb) =>
 {
-    await gulp.src(["assets/**/*.json", "assets/**/*.png", "assets/**/*.ico"]).pipe(gulp.dest(dataDir));
+    await gulp.src(["assets/**/*.json", "assets/**/*.json5", "assets/**/*.png", "assets/**/*.ico"]).pipe(gulp.dest(dataDir));
     await gulp.src([licenseFile]).pipe(rename("LICENSE-Server.txt")).pipe(gulp.dest(buildDir));
     // Write dynamic hashed of asset files for the build 
     const hashFileDir = `${dataDir}\\checks.dat`;

project/package.json

@@ -65,7 +65,8 @@
         "gulp-execa": "5.0.0",
         "gulp-rename": "2.0.0",
         "jest": "29.6.2",
-        "json5": "^2.2.3",
+		"json5": "2.2.3",
+        "jsonc": "2.0.0",
         "madge": "6.1.0",
         "pkg": "5.8.1",
         "pkg-fetch": "3.5.2",

project/src/callbacks/NotifierCallbacks.ts

@@ -8,6 +8,7 @@ import { INotifierChannel } from "../models/eft/notifier/INotifier";
 import { ISelectProfileRequestData } from "../models/eft/notifier/ISelectProfileRequestData";
 import { ISelectProfileResponse } from "../models/eft/notifier/ISelectProfileResponse";
 import { HttpResponseUtil } from "../utils/HttpResponseUtil";
+import { JsonUtil } from "../utils/JsonUtil";
 
 @injectable()
 export class NotifierCallbacks
@@ -15,6 +16,7 @@ export class NotifierCallbacks
     constructor(
         @inject("HttpServerHelper") protected httpServerHelper: HttpServerHelper,
         @inject("HttpResponseUtil") protected httpResponse: HttpResponseUtil,
+        @inject("JsonUtil") protected jsonUtil: JsonUtil,
         @inject("NotifierController") protected notifierController: NotifierController)
     { }
 
@@ -35,7 +37,7 @@ export class NotifierCallbacks
          *  be sent to client as NEWLINE separated strings... yup.
          */
         this.notifierController.notifyAsync(tmpSessionID)
-            .then((messages: any) => messages.map((message: any) => JSON.stringify(message)).join("\n"))
+            .then((messages: any) => messages.map((message: any) => this.jsonUtil.serialize(message)).join("\n"))
             .then((text) => this.httpServerHelper.sendTextJson(resp, text));
     }
 

project/src/controllers/RepeatableQuestController.ts

@@ -1183,7 +1183,7 @@ export class RepeatableQuestController
     protected probabilityObjectArray<K, V>(configArrayInput: ProbabilityObject<K, V>[]): ProbabilityObjectArray<K, V>
     {
         const configArray = this.jsonUtil.clone(configArrayInput);
-        const probabilityArray = new ProbabilityObjectArray<K, V>(this.mathUtil);
+        const probabilityArray = new ProbabilityObjectArray<K, V>(this.mathUtil, this.jsonUtil);
         for (const configObject of configArray)
         {
             probabilityArray.push(new ProbabilityObject(configObject.key, configObject.relativeProbability, configObject.data));

project/src/controllers/TradeController.ts

@@ -22,6 +22,7 @@ import { ConfigServer } from "../servers/ConfigServer";
 import { RagfairServer } from "../servers/RagfairServer";
 import { LocalisationService } from "../services/LocalisationService";
 import { HttpResponseUtil } from "../utils/HttpResponseUtil";
+import { JsonUtil } from "../utils/JsonUtil";
 
 @injectable()
 class TradeController
@@ -35,6 +36,7 @@ class TradeController
         @inject("TradeHelper") protected tradeHelper: TradeHelper,
         @inject("ItemHelper") protected itemHelper: ItemHelper,
         @inject("ProfileHelper") protected profileHelper: ProfileHelper,
+        @inject("JsonUtil") protected jsonUtil: JsonUtil,
         @inject("RagfairServer") protected ragfairServer: RagfairServer,
         @inject("HttpResponseUtil") protected httpResponse: HttpResponseUtil,
         @inject("LocalisationService") protected localisationService: LocalisationService,
@@ -70,7 +72,7 @@ class TradeController
                 return this.httpResponse.appendErrorToOutput(output, errorMessage);
             }
 
-            this.logger.debug(JSON.stringify(offer, null, 2));
+            this.logger.debug(this.jsonUtil.serializeAdvanced(offer, null, 2));
 
             const buyData: IProcessBuyTradeRequestData = {
                 Action: "TradingConfirm",

project/src/generators/LocationGenerator.ts

@@ -167,7 +167,7 @@ export class LocationGenerator
     protected getWeightedCountOfContainerItems(containerTypeId: string, staticLootDist: Record<string, IStaticLootDetails>, locationName: string): number
     {
         // Create probability array to calcualte the total count of lootable items inside container
-        const itemCountArray = new ProbabilityObjectArray<number>(this.mathUtil);
+        const itemCountArray = new ProbabilityObjectArray<number>(this.mathUtil, this.jsonUtil);
         for (const itemCountDistribution of staticLootDist[containerTypeId].itemcountDistribution)
         {
             // Add each count of items into array
@@ -191,7 +191,7 @@ export class LocationGenerator
         const seasonalEventActive = this.seasonalEventService.seasonalEventEnabled();
         const seasonalItemTplBlacklist = this.seasonalEventService.getSeasonalEventItemsToBlock();
 
-        const itemDistribution = new ProbabilityObjectArray<string>(this.mathUtil);
+        const itemDistribution = new ProbabilityObjectArray<string>(this.mathUtil, this.jsonUtil);
         for (const icd of staticLootDist[containerTypeId].itemDistribution)
         {
             if (!seasonalEventActive && seasonalItemTplBlacklist.includes(icd.tpl))
@@ -241,7 +241,7 @@ export class LocationGenerator
             )
         );
 
-        const spawnpointArray = new ProbabilityObjectArray<string, Spawnpoint>(this.mathUtil);
+        const spawnpointArray = new ProbabilityObjectArray<string, Spawnpoint>(this.mathUtil, this.jsonUtil);
         for (const si of dynamicSpawnPoints)
         {
             spawnpointArray.push(
@@ -269,7 +269,7 @@ export class LocationGenerator
         const seasonalItemTplBlacklist = this.seasonalEventService.getSeasonalEventItemsToBlock();
         for (const spawnPoint of spawnPoints)
         {
-            const itemArray = new ProbabilityObjectArray<string>(this.mathUtil);
+            const itemArray = new ProbabilityObjectArray<string>(this.mathUtil, this.jsonUtil);
             for (const itemDist of spawnPoint.itemDistribution)
             {
                 if (!seasonalEventActive && seasonalItemTplBlacklist.includes(spawnPoint.template.Items.find(x => x._id === itemDist.composedKey.key)._tpl))
@@ -320,7 +320,7 @@ export class LocationGenerator
                 }
 
                 // Create probability array of all spawn positions for this spawn id
-                const spawnpointArray = new ProbabilityObjectArray<string, SpawnpointsForced>(this.mathUtil);
+                const spawnpointArray = new ProbabilityObjectArray<string, SpawnpointsForced>(this.mathUtil, this.jsonUtil);
                 for (const si of items)
                 {
                     // use locationId as template.Id is the same across all items

project/src/helpers/ItemHelper.ts

@@ -954,7 +954,7 @@ class ItemHelper
 
     protected drawAmmoTpl(caliber: string, staticAmmoDist: Record<string, IStaticAmmoDetails[]>): string
     {
-        const ammoArray = new ProbabilityObjectArray<string>(this.mathUtil);
+        const ammoArray = new ProbabilityObjectArray<string>(this.mathUtil, this.jsonUtil);
         for (const icd of staticAmmoDist[caliber])
         {
             ammoArray.push(

project/src/loaders/PreAkiModLoader.ts

@@ -87,14 +87,16 @@ export class PreAkiModLoader implements IModLoader
         if (!this.vfs.exists(this.modOrderPath)) 
         {
             this.logger.info(this.localisationService.getText("modloader-mod_order_missing"));
-            this.vfs.writeFile(this.modOrderPath, JSON.stringify({order: []}, null, 4));
+
+            // Write file with empty order array to disk
+            this.vfs.writeFile(this.modOrderPath, this.jsonUtil.serializeAdvanced({order: []}, null, 4));
         }
         else 
         {
             const modOrder = this.vfs.readFile(this.modOrderPath, { encoding: "utf8" });
             try 
             {
-                JSON.parse(modOrder).order.forEach((mod: string, index: number) => 
+                this.jsonUtil.deserialize<any>(modOrder).order.forEach((mod: string, index: number) => 
                 {
                     this.order[mod] = index;
                 });

project/src/routers/serializers/NotifySerializer.ts

@@ -4,6 +4,7 @@ import { inject, injectable } from "tsyringe";
 import { NotifierController } from "../../controllers/NotifierController";
 import { Serializer } from "../../di/Serializer";
 import { HttpServerHelper } from "../../helpers/HttpServerHelper";
+import { JsonUtil } from "../../utils/JsonUtil";
 
 @injectable()
 export class NotifySerializer extends Serializer
@@ -11,6 +12,7 @@ export class NotifySerializer extends Serializer
 
     constructor(
         @inject("NotifierController") protected notifierController: NotifierController,
+        @inject("JsonUtil") protected jsonUtil: JsonUtil,
         @inject("HttpServerHelper") protected httpServerHelper: HttpServerHelper
     )
     {
@@ -28,7 +30,7 @@ export class NotifySerializer extends Serializer
          *  be sent to client as NEWLINE separated strings... yup.
          */
         this.notifierController.notifyAsync(tmpSessionID)
-            .then((messages: any) => messages.map((message: any) => JSON.stringify(message)).join("\n"))
+            .then((messages: any) => messages.map((message: any) => this.jsonUtil.serialize(message)).join("\n"))
             .then((text) => this.httpServerHelper.sendTextJson(resp, text));
     }
 

project/src/servers/ConfigServer.ts

@@ -2,14 +2,15 @@ import { JsonUtil } from "../utils/JsonUtil";
 import { VFS } from "../utils/VFS";
 
 import { inject, injectable } from "tsyringe";
-import { ILogger } from "../models/spt/utils/ILogger";
 import { ConfigTypes } from "../models/enums/ConfigTypes";
 import { ICoreConfig } from "../models/spt/config/ICoreConfig";
+import { ILogger } from "../models/spt/utils/ILogger";
 
 @injectable()
 export class ConfigServer 
 {
     protected configs: Record<string, any> = {};
+    protected readonly acceptableFileExtensions: string[] = ["json", "jsonc"];
 
     constructor(
         @inject("WinstonLogger") protected logger: ILogger,
@@ -34,18 +35,20 @@ export class ConfigServer
     {
         this.logger.debug("Importing configs...");
 
-        // get all filepaths
+        // Get all filepaths
-        const filepath = (globalThis.G_RELEASE_CONFIGURATION) ? "Aki_Data/Server/configs/" : "./assets/configs/";
+        const filepath = (globalThis.G_RELEASE_CONFIGURATION) 
+            ? "Aki_Data/Server/configs/"
+            : "./assets/configs/";
         const files = this.vfs.getFiles(filepath);
 
-        // add file content to result
+        // Add file content to result
         for (const file of files)
         {
-            if (this.vfs.getFileExtension(file) === "json")
+            if (this.acceptableFileExtensions.includes(this.vfs.getFileExtension(file.toLowerCase())))
             {
-                const filename = this.vfs.stripExtension(file);
+                const fileName = this.vfs.stripExtension(file);
                 const filePathAndName = `${filepath}${file}`;
-                this.configs[`aki-${filename}`] = this.jsonUtil.deserializeWithCacheCheck(this.vfs.readFile(filePathAndName), filePathAndName);
+                this.configs[`aki-${fileName}`] = this.jsonUtil.deserializeJsonC<any>(this.vfs.readFile(filePathAndName), filePathAndName);
             }
         }
         

project/src/servers/WebSocketServer.ts

@@ -8,6 +8,7 @@ import { ConfigTypes } from "../models/enums/ConfigTypes";
 import { IHttpConfig } from "../models/spt/config/IHttpConfig";
 import { ILogger } from "../models/spt/utils/ILogger";
 import { LocalisationService } from "../services/LocalisationService";
+import { JsonUtil } from "../utils/JsonUtil";
 import { RandomUtil } from "../utils/RandomUtil";
 import { ConfigServer } from "./ConfigServer";
 
@@ -20,6 +21,7 @@ export class WebSocketServer
         @inject("RandomUtil") protected randomUtil: RandomUtil,
         @inject("ConfigServer") protected configServer: ConfigServer,
         @inject("LocalisationService") protected localisationService: LocalisationService,
+        @inject("JsonUtil") protected jsonUtil: JsonUtil,
         @inject("HttpServerHelper") protected httpServerHelper: HttpServerHelper
     ) 
     {
@@ -56,7 +58,7 @@ export class WebSocketServer
         {
             if (this.isConnectionWebSocket(sessionID))
             {
-                this.webSockets[sessionID].send(JSON.stringify(output));
+                this.webSockets[sessionID].send(this.jsonUtil.serialize(output));
                 this.logger.debug(this.localisationService.getText("websocket-message_sent"));
             }
             else
@@ -141,7 +143,7 @@ export class WebSocketServer
 
             if (ws.readyState === WebSocket.OPEN) 
             {
-                ws.send(JSON.stringify(this.defaultNotification));
+                ws.send(this.jsonUtil.serialize(this.defaultNotification));
             }
             else 
             {

project/src/servers/http/AkiHttpListener.ts

@@ -126,14 +126,14 @@ export class AkiHttpListener implements IHttpListener
             let data: any;
             try
             {
-                data = JSON.parse(output);
+                data = this.jsonUtil.deserialize(output);
             }
             catch (e)
             {
                 data = output;
             }
             const log = new Response(req.method, data);
-            this.requestsLogger.info(`RESPONSE=${JSON.stringify(log)}`);
+            this.requestsLogger.info(`RESPONSE=${this.jsonUtil.serialize(log)}`);
         }
     }
     
@@ -145,10 +145,10 @@ export class AkiHttpListener implements IHttpListener
             // Parse quest info into object
             const data = (typeof info === "object")
                 ? info
-                : JSON.parse(info);
+                : this.jsonUtil.deserialize(info);
 
             const log = new Request(req.method, new RequestData(req.url, req.headers, data));
-            this.requestsLogger.info(`REQUEST=${JSON.stringify(log)}`);
+            this.requestsLogger.info(`REQUEST=${this.jsonUtil.serialize(log)}`);
         }
         
         let output = this.httpRouter.getResponse(req, info, sessionID);

project/src/utils/DatabaseImporter.ts

@@ -63,7 +63,7 @@ export class DatabaseImporter implements OnLoad
                 const fileWithPath = `${this.filepath}${file}`;
                 if (this.vfs.exists(fileWithPath))
                 {
-                    this.hashedFile = this.jsonUtil.deserialize(this.encodingUtil.fromBase64(this.vfs.readFile(fileWithPath)));
+                    this.hashedFile = this.jsonUtil.deserialize(this.encodingUtil.fromBase64(this.vfs.readFile(fileWithPath)), file);
                 }
                 else
                 {

project/src/utils/JsonUtil.ts

@@ -1,6 +1,7 @@
 import fixJson from "json-fixer";
+import { jsonc } from "jsonc";
+import { IParseOptions, IStringifyOptions, Reviver } from "jsonc/lib/interfaces";
 import { inject, injectable } from "tsyringe";
-
 import { ILogger } from "../models/spt/utils/ILogger";
 import { HashUtil } from "./HashUtil";
 import { VFS } from "./VFS";
@@ -10,6 +11,7 @@ export class JsonUtil
 {
     protected fileHashes = null;
     protected jsonCacheExists = false;
+    protected jsonCachePath = "./user/cache/jsonCache.json";
 
     constructor(
         @inject("VFS") protected vfs: VFS,
@@ -21,10 +23,10 @@ export class JsonUtil
     /**
      * From object to string
      * @param data object to turn into JSON
-     * @param prettify Should output be prettified?
+     * @param prettify Should output be prettified
      * @returns string
      */
-    public serialize<T>(data: T, prettify = false): string
+    public serialize(data: any, prettify = false): string
     {
         if (prettify)
         {
@@ -36,20 +38,78 @@ export class JsonUtil
         }
     }
 
+    /**
+     * From object to string
+     * @param data object to turn into JSON
+     * @param replacer An array of strings and numbers that acts as an approved list for selecting the object properties that will be stringified.
+     * @param space Adds indentation, white space, and line break characters to the return-value JSON text to make it easier to read.
+     * @returns string
+     */
+    public serializeAdvanced(data: any, replacer?: (this: any, key: string, value: any) => any, space?: string | number): string
+    {
+
+        return JSON.stringify(data, replacer, space);
+    }
+
+    /**
+     * From object to string
+     * @param data object to turn into JSON
+    * @param filename Name of file being serialized
+    * @param options Stringify options or a replacer.
+    * @returns The string converted from the JavaScript value
+     */
+    public serializeJsonC(
+        data: any,
+        filename?: string | null,
+        options?: IStringifyOptions | Reviver): string
+    {
+        try
+        {
+            return jsonc.stringify(data, options);
+        }
+        catch (error)
+        {
+            this.logger.error(`unable to stringify jsonC file: ${filename} message: ${error.message}, stack: ${error.stack}`);
+        }
+        
+    }
+
     /**
      * From string to object
      * @param jsonString json string to turn into object
+     * @param filename Name of file being deserialized
      * @returns object
      */
     public deserialize<T>(jsonString: string, filename = ""): T
     {
-        const { data, changed } = fixJson(`${jsonString}`);
+        try
-        if (changed)
         {
-            this.logger.error(`Invalid JSON ${filename} was detected and automatically fixed, please ensure any edits performed recently are valid, always run your JSON through an online JSON validator prior to starting the server`);
+            return JSON.parse(jsonString);
+        }
+        catch (error)
+        {
+            this.logger.error(`unable to parse json file: ${filename} message: ${error.message}, stack: ${error.stack}`);
+        }
+    }
+
+    /**
+     * From string to object
+     * @param jsonString json string to turn into object
+     * @param filename Name of file being deserialized
+     * @param options Parsing options
+     * @returns object
+     */
+    public deserializeJsonC<T>(jsonString: string, filename = "", options?: IParseOptions): T
+    {
+        try
+        {
+            return jsonc.parse(jsonString, options);
+        }
+        catch (error)
+        {
+            this.logger.error(`unable to parse jsonC file: ${filename} message: ${error.message}, stack: ${error.stack}`);
         }
         
-        return data;
     }
 
     public async deserializeWithCacheCheckAsync<T>(jsonString: string, filePath: string): Promise<T>
@@ -60,30 +120,20 @@ export class JsonUtil
         });
     }
 
+    /**
+     * From json string to object
+     * @param jsonString String to turn into object
+     * @param filePath Path to json file being processed
+     * @returns Object
+     */
     public deserializeWithCacheCheck<T>(jsonString: string, filePath: string): T
     {
-        // get json cache file and ensure it exists, create if it doesnt
+        this.ensureJsonCacheExists(this.jsonCachePath);
-        const jsonCachePath = "./user/cache/jsonCache.json";
+        this.hydrateJsonCache(this.jsonCachePath);
-        if (!this.jsonCacheExists)
-        {
-            
-            if (!this.vfs.exists(jsonCachePath))
-            {
-                this.vfs.writeFile(jsonCachePath, "{}");
-            }
-            this.jsonCacheExists = true;
-        }
-
 
         // Generate hash of string
         const generatedHash = this.hashUtil.generateSha1ForData(jsonString);
 
-        // Get all file hashes
-        if (!this.fileHashes)
-        {
-            this.fileHashes = this.deserialize(this.vfs.readFile(`${jsonCachePath}`));
-        }
-
         // Get hash of file and check if missing or hash mismatch
         let savedHash = this.fileHashes[filePath];
         if (!savedHash || savedHash !== generatedHash)
@@ -99,7 +149,7 @@ export class JsonUtil
                 {
                     // data valid, save hash and call function again
                     this.fileHashes[filePath] = generatedHash;
-                    this.vfs.writeFile(jsonCachePath, this.serialize(this.fileHashes, true));
+                    this.vfs.writeFile(this.jsonCachePath, this.serialize(this.fileHashes, true));
                     savedHash = generatedHash;
                 }
                 return data as T;
@@ -119,11 +169,47 @@ export class JsonUtil
         }
 
         // Match!
-        return JSON.parse(jsonString) as T;
+        return this.deserialize<T>(jsonString);
     }
 
-    public clone<T>(data: T): T
+    
+    /**
+     * Create file if nothing found
+     * @param jsonCachePath path to cache
+     */
+    protected ensureJsonCacheExists(jsonCachePath: string): void
     {
-        return JSON.parse(JSON.stringify(data));
+        if (!this.jsonCacheExists)
+        {
+            if (!this.vfs.exists(jsonCachePath))
+            {
+                // Create empty object at path
+                this.vfs.writeFile(jsonCachePath, "{}");
+            }
+            this.jsonCacheExists = true;
+        }
+    }
+
+    /**
+     * Read contents of json cache and add to class field
+     * @param jsonCachePath Path to cache
+     */
+    protected hydrateJsonCache(jsonCachePath: string) : void
+    {
+        // Get all file hashes
+        if (!this.fileHashes)
+        {
+            this.fileHashes = this.deserialize(this.vfs.readFile(`${jsonCachePath}`));
+        }
+    }
+
+    /**
+     * Convert into string and back into object to clone object
+     * @param objectToClone Item to clone
+     * @returns Cloned parameter
+     */
+    public clone<T>(objectToClone: T): T
+    {
+        return this.deserialize<T>(this.serialize(objectToClone));
     }
 }

project/src/utils/RandomUtil.ts

@@ -23,6 +23,7 @@ export class ProbabilityObjectArray<K, V=undefined> extends Array<ProbabilityObj
 {
     constructor(
         private mathUtil: MathUtil,
+        private jsonUtil: JsonUtil,
         ...items: ProbabilityObject<K, V>[]) 
     {
         super();
@@ -31,7 +32,7 @@ export class ProbabilityObjectArray<K, V=undefined> extends Array<ProbabilityObj
 
     filter(callbackfn: (value: ProbabilityObject<K, V>, index: number, array: ProbabilityObject<K, V>[]) => any): ProbabilityObjectArray<K, V>
     {
-        return new ProbabilityObjectArray(this.mathUtil, ...super.filter(callbackfn));
+        return new ProbabilityObjectArray(this.mathUtil, this.jsonUtil, ...super.filter(callbackfn));
     }
 
     /**
@@ -53,8 +54,8 @@ export class ProbabilityObjectArray<K, V=undefined> extends Array<ProbabilityObj
      */
     clone(): ProbabilityObjectArray<K, V>
     {
-        const clone = JSON.parse(JSON.stringify(this));
+        const clone = this.jsonUtil.clone(this);
-        const probabliltyObjects = new ProbabilityObjectArray<K, V>(this.mathUtil);
+        const probabliltyObjects = new ProbabilityObjectArray<K, V>(this.mathUtil, this.jsonUtil);
         for (const ci of clone) 
         {
             probabliltyObjects.push(new ProbabilityObject(ci.key, ci.relativeProbability, ci.data));