Rewrites CacheService To Clone Server

The CacheService has been rewritten to clone/pull the server repo for direct access to the data files that are needed to build the cache.

.dockerignore

@@ -0,0 +1,16 @@
+api/node_modules
+api/cache-repo
+frontend/node_modules
+frontend/dist
+.git
+.gitignore
+.env
+.vscode
+.idea
+*.swp
+*.iml
+Thumbs.db
+Desktop.ini
+.DS_Store
+coverage
+*.lcov

.gitignore

@@ -39,3 +39,4 @@ yarn-error.log*
 # Okteto
 **/okteto.yml
 **/.stignore
+api/cache-repo

api/.env.example

@@ -1,4 +1,8 @@
-GITEA_ITEMS_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/items.json"
+GIT_REPO_URL="https://github.com/sp-tarkov/server.git"
-GITEA_CUSTOMIZATION_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/customization.json"
+
-GITEA_HANDBOOK_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/handbook.json"
+ITEMS_PATH="project/assets/database/templates/items.json"
-GITEA_LOCALES_API_URL="https://dev.sp-tarkov.com/api/v1/repos/SPT/server-mirror/contents/project%2Fassets%2Fdatabase%2Flocales%2Fglobal"
+CUSTOMIZATION_PATH="project/assets/database/templates/customization.json"
+HANDBOOK_PATH="project/assets/database/templates/handbook.json"
+
+TEMPLATES_PATH="project/assets/database/templates"
+LOCALES_PATH="project/assets/database/locales/global"

api/package.json

@@ -12,7 +12,8 @@
     "@elysiajs/swagger": "^1.1.6",
     "axios": "^1.7.9",
     "elysia": "^1.1.26",
-    "logestic": "^1.2.4"
+    "logestic": "^1.2.4",
+    "simple-git": "^3.27.0"
   },
   "devDependencies": {
     "@types/bun": "latest"

api/src/services/CacheService.ts

@@ -1,48 +1,95 @@
-import axios from "axios";
 import { ILocalesCache, IHandbookCache, ICache, CacheKeys, CacheValues, ILocaleCache } from "@src/interfaces/ICaches";
-import { IGiteaContentsResponse } from "@src/interfaces/IGiteaContentsResponse";
+import simpleGit, { SimpleGit } from "simple-git";
-import { IGiteaItem } from "@src/interfaces/IGiteaItem";
+import fs from "fs/promises";
-import { IGiteaLocale } from "@src/interfaces/IGiteaLocale";
+import path from "path";
-import { IGiteaHandbook } from "@src/interfaces/IGiteaHandbook";
-import { promisify } from "util";
 
 export class CacheService {
     private static cache: Map<CacheKeys, CacheValues> = new Map();
+    private static repoPath = path.resolve("cache-repo");
 
-    private static async refreshItemsCache(): Promise<void> {
+    // Pull or clone the repository
-        const items = (await axios.get<Record<string, IGiteaItem>>(Bun.env.GITEA_ITEMS_URL)).data;
+    private static async pullRepository(): Promise<void> {
-        const customization = (await axios.get<Record<string, IGiteaItem>>(Bun.env.GITEA_CUSTOMIZATION_URL)).data;
+        const git: SimpleGit = simpleGit();
 
-        this.setCache("items", {
+        if (!(await fs.stat(this.repoPath).catch(() => false))) {
-            ...items,
+            console.log("[CACHE] Cloning repository with sparse checkout...");
-            ...customization
+            await git.clone(<string>Bun.env.GIT_REPO_URL, this.repoPath, ["--depth", "1", "--filter=blob:none"]);
-        });
+
+            // Enable sparse checkout
+            console.log("[CACHE] Initializing sparse checkout...");
+            await git.cwd(this.repoPath).raw(["sparse-checkout", "init", "--cone"]);
+
+            // Define the directories and specific files to include
+            const sparsePaths = [
+                <string>Bun.env.TEMPLATES_PATH, // Includes items.json and customization.json
+                <string>Bun.env.LOCALES_PATH,  // Includes all locales files
+            ];
+
+            console.log(`[CACHE] Setting sparse-checkout paths: ${sparsePaths.join(", ")}`);
+            await git.cwd(this.repoPath).raw(["sparse-checkout", "set", ...sparsePaths]);
+        } else {
+            console.log("[CACHE] Pulling updates with sparse checkout...");
+            await git.cwd(this.repoPath).pull();
         }
 
+        console.log("[CACHE] Fetching specific LFS-managed files...");
+        try {
+            // Fetch and pull only the required LFS-managed file(s)
+            await git.cwd(this.repoPath).raw(["lfs", "fetch", "--include", <string>Bun.env.ITEMS_PATH]);
+            await git.cwd(this.repoPath).raw(["lfs", "pull", "--include", <string>Bun.env.ITEMS_PATH]);
+        } catch (error) {
+            console.error("[CACHE] Error fetching LFS data:", error);
+        }
+    }
+
+    // Refresh Items Cache
+    private static async refreshItemsCache(): Promise<void> {
+        const itemsPath = path.join(this.repoPath, <string>Bun.env.ITEMS_PATH);
+        const customizationPath = path.join(this.repoPath, <string>Bun.env.CUSTOMIZATION_PATH);
+
+        console.log(`[CACHE] Loading items from ${itemsPath}`);
+        console.log(`[CACHE] Loading customization from ${customizationPath}`);
+
+        try {
+            const items = JSON.parse(await fs.readFile(itemsPath, "utf-8"));
+            const customization = JSON.parse(await fs.readFile(customizationPath, "utf-8"));
+
+            this.setCache("items", { ...items, ...customization });
+        } catch (error) {
+            console.error("[CACHE] Error refreshing items cache:", error);
+        }
+    }
+
+    // Refresh Locales Cache
     private static async refreshLocalesCache(): Promise<void> {
+        const localesDir = path.join(this.repoPath, <string>Bun.env.LOCALES_PATH);
+        console.log(`[CACHE] Scanning locales directory: ${localesDir}`);
+
+        try {
+            const localeFiles = await fs.readdir(localesDir);
             let locales: ILocalesCache = {};
 
-        const localesList = (await axios.get<IGiteaContentsResponse[]>(Bun.env.GITEA_LOCALES_API_URL)).data;
+            for (const fileName of localeFiles) {
-
+                if (!fileName.endsWith(".json")) {
-        for (const locale of localesList) {
+                    console.log(`[CACHE] Skipping non-JSON file: ${fileName}`);
-            const localeName = locale.name.split(".json")[0].trim();
-            if (localeName.length === 0) {
                     continue;
                 }
 
-            const localeJson = (await axios.get<IGiteaLocale>(locale.download_url)).data;
+                const localeName = fileName.split(".json")[0].trim(); // Extract locale name from file name
+                console.log(`[CACHE] Processing locale file: ${fileName} (${localeName})`);
+
+                const localeJson = JSON.parse(await fs.readFile(path.join(localesDir, fileName), "utf-8"));
 
                 const templates: ILocaleCache = {};
-
                 for (const item of Object.values(this.getCache("items"))) {
                     if (item._type === "Node" || localeJson[`${item._id} Name`] === undefined) {
                         continue;
                     }
 
                     templates[item._id] = {
-                    "Name": localeJson[`${item._id} Name`],
+                        Name: localeJson[`${item._id} Name`],
-                    "ShortName": localeJson[`${item._id} ShortName`],
+                        ShortName: localeJson[`${item._id} ShortName`],
-                    "Description": localeJson[`${item._id} Description`]
+                        Description: localeJson[`${item._id} Description`],
                     };
                 }
 
@@ -50,24 +97,38 @@ export class CacheService {
             }
 
             this.setCache("locales", locales);
+            console.log(`[CACHE] Locales cache refreshed successfully.`);
+        } catch (error) {
+            console.error("[CACHE] Error refreshing locales cache:", error);
+        }
     }
 
+    // Refresh Handbook Cache
     private static async refreshHandbookCache(): Promise<void> {
-        let handbook: IHandbookCache = {};
+        const handbookPath = path.join(this.repoPath, <string>Bun.env.HANDBOOK_PATH);
-        const gitea_handbook = (await axios.get<{ Items: IGiteaHandbook[] }>(Bun.env.GITEA_HANDBOOK_URL)).data;
+        console.log(`[CACHE] Loading handbook from ${handbookPath}`);
 
-        for (const handbookItem of gitea_handbook.Items) {
+        try {
+            const giteaHandbook = JSON.parse(await fs.readFile(handbookPath, "utf-8"));
+
+            const handbook: IHandbookCache = {};
+            for (const handbookItem of giteaHandbook.Items) {
                 handbook[handbookItem.Id] = {
                     ParentId: handbookItem.ParentId,
-                Price: handbookItem.Price
+                    Price: handbookItem.Price,
                 };
             }
 
             this.setCache("handbook", handbook);
+        } catch (error) {
+            console.error("[CACHE] Error refreshing handbook cache:", error);
+        }
     }
 
+    // Refresh All Caches
     public static async refreshAllCache(): Promise<void> {
         console.log("[CACHE] Refreshing cache...");
+        await this.pullRepository();
         await this.refreshItemsCache();
         await this.refreshLocalesCache();
         await this.refreshHandbookCache();