From 9b44ca6d26a049d4e4547b49326ec447ed973b49 Mon Sep 17 00:00:00 2001 From: Refringe Date: Sat, 7 Dec 2024 21:33:03 -0500 Subject: [PATCH] Rewrites CacheService To Clone Server The CacheService has been rewritten to clone/pull the server repo for direct access to the data files that are needed to build the cache. --- .dockerignore | 16 ++++ .gitignore | 3 +- api/.env.example | 12 ++- api/bun.lockb | Bin 13331 -> 15157 bytes api/package.json | 3 +- api/src/services/CacheService.ts | 157 +++++++++++++++++++++---------- 6 files changed, 137 insertions(+), 54 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..160ffcc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +api/node_modules +api/cache-repo +frontend/node_modules +frontend/dist +.git +.gitignore +.env +.vscode +.idea +*.swp +*.iml +Thumbs.db +Desktop.ini +.DS_Store +coverage +*.lcov diff --git a/.gitignore b/.gitignore index bca44b5..5cea0ff 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,5 @@ yarn-error.log* # Okteto **/okteto.yml -**/.stignore \ No newline at end of file +**/.stignore +api/cache-repo diff --git a/api/.env.example b/api/.env.example index 1980c95..d4d199c 100644 --- a/api/.env.example +++ b/api/.env.example @@ -1,4 +1,8 @@ -GITEA_ITEMS_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/items.json" -GITEA_CUSTOMIZATION_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/customization.json" -GITEA_HANDBOOK_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/handbook.json" -GITEA_LOCALES_API_URL="https://dev.sp-tarkov.com/api/v1/repos/SPT/server-mirror/contents/project%2Fassets%2Fdatabase%2Flocales%2Fglobal" +GIT_REPO_URL="https://github.com/sp-tarkov/server.git" + +ITEMS_PATH="project/assets/database/templates/items.json" +CUSTOMIZATION_PATH="project/assets/database/templates/customization.json" +HANDBOOK_PATH="project/assets/database/templates/handbook.json" + +TEMPLATES_PATH="project/assets/database/templates" +LOCALES_PATH="project/assets/database/locales/global" diff --git a/api/bun.lockb b/api/bun.lockb index 6303c83e9fcc4fef9ea58fedab35b9344bdbad42..0ee548a4df4d95556f67dc4ed709bc857e495a07 100755 GIT binary patch delta 3223 zcmbtWeN>a@6@TBCBoH)!K!5;B00{&n;j_UEGbXOOMdwGI(XOWxNhFyBsG%0|bl}=L zN@>+z=d_AnU3;9LXH&Ytj#W3?Q&hB^b$;MFTRpDS&*Emc=1jWZeZxCF9{+K@IZuAi z@AfyaP6 z(C_nm^4*JsF7RmZHg#xRRbY8t^%7rW4BBJRUW7GxgNCX`FZL5QX+nx*h6{oo?Rp@u z-|Y2NxzX>1VpWT3JsGIC@B-e55zT7r>v#cJD^GwDvLdsf6)V<>zfr(Tz|p{U540O? z9nCYZwVxfmTc5V9x8!6nenYVP>YNR89PKroZ=cJ4?FFIa)8ymt|LWkezjU2lJ%g1h4@KP!Y%~cAh5=LPH&PybAw{Tv2R57TMwCa0f-n``B8d?;?MyJ3+^V9Z8k=?{ z*aR?%R)*Q6y&?rAhjc|GdzeF-Em1RIokTr=n-T>98DV5sIi!X#Y6k2HqaMKD!YBx^ zsmQK&NGnv-3^=8t9>6Fy1p$-QWY;*f>v0HEWL;LOZPI|6dNd9xRzpF+k2GWtcSt)m z)C?HVP!FIioPvO#g_AwPp*@1LVC5weq3-Q)>Oo^J{O^$0Wc!wam4m5iWw=ed4=gl+ z*(YGQy&+FX$yy3(9nu0V*(0&7mYM-KwbTQ!Mp6(^9ZB|4_7)rzpP;}F#>vUT2BxYllHESb&H$*PkFYVo?X8B$d{>|7r)Ld%u!4I zhr4fS|C)(o6@}sGVIbd8MO2$Eeznm3?Cm)P@1<8B9kOkUy71)HLu=3MyE>@*O`^?hw-WVMn0@IQ<+C@oY zbmWhAile9->;tf|F-}oOz8DuZ#pmzO z#p~!}Fg=;$TvVpl(W`MzF_F%L4T4RIcZvpjIo?GZ5_EI}EQKcOT{JCGN1b}7IF>#K z`#0E(1gDrr8xvf#Gf78x!P4n_i7uLN(9zyRrQ6 z;Q}2oxDS*hUyh(z$#)`WaSVVok&D7W~LL2(T!8=g5>PX8Qhk6a7BZ z*n*%d@Qvpo9Hi1vj3^ut6H$^-I9kS{#G!Bm2;Q-0OlH*W=S->X)QjiOMGFmul^BUIBAgWP;BCSfc3>#r)(@Q!iuv2in8n!Za0n~8_5n+ zjLJrxja$f`gj855BkRkSN>rgstCBNFth6jjzNKUhlK6CqrPxv`2<_u~p;OWlOe(^1#gt&0B_&JbH5EyD5^XjmNmC?x$5ix~ zl6j?Mgl(O@t>j8*n^MbTf)W0PTU%Feoa()uvdfB|5=()lfU`UrmtB;qBy%&L%iUVq zzkgVsT3{&?gkvhI%`TGK)U+)-DOE}0UcWH&$EB<0PQ*AYV<{{`o=G2NPft~{y2CXC zn@)GXJV9P)BD*IuoXT^iOWp`-&sm(RByTIO*Yvf;Udxa~_EoV>5Y}mFv^gnN$?Dcx zpM2HSH3>WYw1_vdlwHzN#X|>-9H9^z5V?jwl#matcxCal%WT0R(iwA6vK!YBx32ZY zveJ`cQ`E2kN3e9-DI?bx8SwcVYCZWizDD{^amvhl)S7R-NAim;MK;*pSkqjw{+_hCi)Y+_fB3$L@zf4cO}v!VetI3=gzy+9)jW1>m*ryk6h7iLL#cGh;`gzYzNgoEiX*_KgkU zP!ad!txW^Vv47k{y|Px#710h^+xne1Z>7yA#5}wm2NnaPK(=1kuTSpZvF=*-tCQE| z2B}sF%DPRpQ&7%rYTv`kD(Du|dUsMqC8a#3+AApwI4vm$C{pAvGF7*t6ktqI7BHhI z2MD;x?={sX7o`Aix+n{{;-Z|_)S3}w4~~`;@FulUunsUm0T?>wrmW9Yzqu&~Fg@hg zOf~AElxAw5BiNL!Ips>KU=ii8`ACrv$j%-`Q}}$ch*Eyja}Jv(9dLE|kclq5^-w@d zYCT|iT`4ab^WP z;{Ri>|AYK}sq-Bp^9qQBR6=+$G7mDr0`4w6+(INzgl(2b)QhNfI3x3Vhu@C8zZk*> z8X&wFnKwFk50Lflb$F9wUo<%z5eETwR_=Oc^6~#?YS@R{Qzh<3W+Hb^jcxzm%+SPw zF?u^-QO81^{t8&qMGuyw=?Ad)N-XK2n_w>obb7qhl3seJG)=2ZbkfT#snMflY5D`~ zBd~d-m8a=gsZPV?megq)Y(tq&H5HazK$(g(`O0;g0Si!7WtvWdJy&VTQu+*Rs6wZf zpe4&`BABLNrB2^~RZ^rXO&7opR9Ui$z6Ki)>a?=jk|Ekxou;NLoqhqUq2-}8T>*P7 zWXU@E5p1$rr-#CpTttV$Y3c|ezVM}!t+iKu{wJMMnrxtwrtX-S`#ZllIg21%U0hYX z3jDP5&HatEHnN!BYHHlgS+0k0<)fF1Y6#yhbU;x8;hTqk#{Jf=J3ev=gbSK4Lk)y) zRT%``(Jo-V40RB`ReaO9hDX^DR?zXu-}Ove!mOvqBIS0ap+VSZ@tN!g+eZJiF9nCf z_0BqcNzifZZop;df^gvMFZ+s4Z~Ko4aLaFWd)q(OWncLW_KW>R+)3L}4xC?v9ONK4 zZyZQHgac+=?F6ttLY>jy>EeBB)d@l2=7c&g=;7v|nicdk*il7CnuFoO6ng2<14B1L z@5YpTVyV$;v4k3D7RNs;a7v@yi_^wdn8{fXF zUYE=})iwhvlvv = new Map(); + private static repoPath = path.resolve("cache-repo"); - private static async refreshItemsCache(): Promise { - const items = (await axios.get>(Bun.env.GITEA_ITEMS_URL)).data; - const customization = (await axios.get>(Bun.env.GITEA_CUSTOMIZATION_URL)).data; + // Pull or clone the repository + private static async pullRepository(): Promise { + const git: SimpleGit = simpleGit(); - this.setCache("items", { - ...items, - ...customization - }); + if (!(await fs.stat(this.repoPath).catch(() => false))) { + console.log("[CACHE] Cloning repository with sparse checkout..."); + await git.clone(Bun.env.GIT_REPO_URL, this.repoPath, ["--depth", "1", "--filter=blob:none"]); + + // Enable sparse checkout + console.log("[CACHE] Initializing sparse checkout..."); + await git.cwd(this.repoPath).raw(["sparse-checkout", "init", "--cone"]); + + // Define the directories and specific files to include + const sparsePaths = [ + Bun.env.TEMPLATES_PATH, // Includes items.json and customization.json + Bun.env.LOCALES_PATH, // Includes all locales files + ]; + + console.log(`[CACHE] Setting sparse-checkout paths: ${sparsePaths.join(", ")}`); + await git.cwd(this.repoPath).raw(["sparse-checkout", "set", ...sparsePaths]); + } else { + console.log("[CACHE] Pulling updates with sparse checkout..."); + await git.cwd(this.repoPath).pull(); + } + + console.log("[CACHE] Fetching specific LFS-managed files..."); + try { + // Fetch and pull only the required LFS-managed file(s) + await git.cwd(this.repoPath).raw(["lfs", "fetch", "--include", Bun.env.ITEMS_PATH]); + await git.cwd(this.repoPath).raw(["lfs", "pull", "--include", Bun.env.ITEMS_PATH]); + } catch (error) { + console.error("[CACHE] Error fetching LFS data:", error); + } } + // Refresh Items Cache + private static async refreshItemsCache(): Promise { + const itemsPath = path.join(this.repoPath, Bun.env.ITEMS_PATH); + const customizationPath = path.join(this.repoPath, Bun.env.CUSTOMIZATION_PATH); + + console.log(`[CACHE] Loading items from ${itemsPath}`); + console.log(`[CACHE] Loading customization from ${customizationPath}`); + + try { + const items = JSON.parse(await fs.readFile(itemsPath, "utf-8")); + const customization = JSON.parse(await fs.readFile(customizationPath, "utf-8")); + + this.setCache("items", { ...items, ...customization }); + } catch (error) { + console.error("[CACHE] Error refreshing items cache:", error); + } + } + + // Refresh Locales Cache private static async refreshLocalesCache(): Promise { - let locales: ILocalesCache = {}; + const localesDir = path.join(this.repoPath, Bun.env.LOCALES_PATH); + console.log(`[CACHE] Scanning locales directory: ${localesDir}`); - const localesList = (await axios.get(Bun.env.GITEA_LOCALES_API_URL)).data; + try { + const localeFiles = await fs.readdir(localesDir); + let locales: ILocalesCache = {}; - for (const locale of localesList) { - const localeName = locale.name.split(".json")[0].trim(); - if (localeName.length === 0) { - continue; - } - - const localeJson = (await axios.get(locale.download_url)).data; - - const templates: ILocaleCache = {}; - - for (const item of Object.values(this.getCache("items"))) { - if (item._type === "Node" || localeJson[`${item._id} Name`] === undefined) { + for (const fileName of localeFiles) { + if (!fileName.endsWith(".json")) { + console.log(`[CACHE] Skipping non-JSON file: ${fileName}`); continue; } - templates[item._id] = { - "Name": localeJson[`${item._id} Name`], - "ShortName": localeJson[`${item._id} ShortName`], - "Description": localeJson[`${item._id} Description`] + const localeName = fileName.split(".json")[0].trim(); // Extract locale name from file name + console.log(`[CACHE] Processing locale file: ${fileName} (${localeName})`); + + const localeJson = JSON.parse(await fs.readFile(path.join(localesDir, fileName), "utf-8")); + + const templates: ILocaleCache = {}; + for (const item of Object.values(this.getCache("items"))) { + if (item._type === "Node" || localeJson[`${item._id} Name`] === undefined) { + continue; + } + + templates[item._id] = { + Name: localeJson[`${item._id} Name`], + ShortName: localeJson[`${item._id} ShortName`], + Description: localeJson[`${item._id} Description`], + }; + } + + locales[localeName] = templates; + } + + this.setCache("locales", locales); + console.log(`[CACHE] Locales cache refreshed successfully.`); + } catch (error) { + console.error("[CACHE] Error refreshing locales cache:", error); + } + } + + // Refresh Handbook Cache + private static async refreshHandbookCache(): Promise { + const handbookPath = path.join(this.repoPath, Bun.env.HANDBOOK_PATH); + console.log(`[CACHE] Loading handbook from ${handbookPath}`); + + try { + const giteaHandbook = JSON.parse(await fs.readFile(handbookPath, "utf-8")); + + const handbook: IHandbookCache = {}; + for (const handbookItem of giteaHandbook.Items) { + handbook[handbookItem.Id] = { + ParentId: handbookItem.ParentId, + Price: handbookItem.Price, }; } - locales[localeName] = templates; + this.setCache("handbook", handbook); + } catch (error) { + console.error("[CACHE] Error refreshing handbook cache:", error); } - - this.setCache("locales", locales); - } - - private static async refreshHandbookCache(): Promise { - let handbook: IHandbookCache = {}; - const gitea_handbook = (await axios.get<{ Items: IGiteaHandbook[] }>(Bun.env.GITEA_HANDBOOK_URL)).data; - - for (const handbookItem of gitea_handbook.Items) { - handbook[handbookItem.Id] = { - ParentId: handbookItem.ParentId, - Price: handbookItem.Price - }; - } - - this.setCache("handbook", handbook); } + // Refresh All Caches public static async refreshAllCache(): Promise { console.log("[CACHE] Refreshing cache..."); + await this.pullRepository(); await this.refreshItemsCache(); await this.refreshLocalesCache(); await this.refreshHandbookCache();