spt/db-website
spt/db-website
0
0
Fork 0
mirror of https://github.com/sp-tarkov/db-website.git synced 2025-02-02 17:49:21 -05:00
Code

Rewrites CacheService To Clone Server

Browse Source 
The CacheService has been rewritten to clone/pull the server repo for direct access to the data files that are needed to build the cache.
This commit is contained in:
Refringe 2024-12-07 21:33:03 -05:00
parent 08911e76de
commit 9b44ca6d26
Signed by: Refringe
SSH Key Fingerprint: SHA256:t865XsQpfTeqPRBMN2G6+N8wlDjkgUCZF3WGW6O9N/k
6 changed files with 137 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.dockerignore Normal file
View File

@ -0,0 +1,16 @@
api/node_modules
api/cache-repo
frontend/node_modules
frontend/dist
.git
.gitignore
.env
.vscode
.idea
*.swp
*.iml
Thumbs.db
Desktop.ini
.DS_Store
coverage
*.lcov

3
.gitignore vendored
View File

@ -38,4 +38,5 @@ yarn-error.log*
# Okteto
**/okteto.yml
**/.stignore
**/.stignore
api/cache-repo

12
api/.env.example
View File

@ -1,4 +1,8 @@
GITEA_ITEMS_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/items.json"
GITEA_CUSTOMIZATION_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/customization.json"
GITEA_HANDBOOK_URL="https://raw.githubusercontent.com/sp-tarkov/server/refs/heads/master/project/assets/database/templates/handbook.json"
GITEA_LOCALES_API_URL="https://dev.sp-tarkov.com/api/v1/repos/SPT/server-mirror/contents/project%2Fassets%2Fdatabase%2Flocales%2Fglobal"
GIT_REPO_URL="https://github.com/sp-tarkov/server.git"
ITEMS_PATH="project/assets/database/templates/items.json"
CUSTOMIZATION_PATH="project/assets/database/templates/customization.json"
HANDBOOK_PATH="project/assets/database/templates/handbook.json"
TEMPLATES_PATH="project/assets/database/templates"
LOCALES_PATH="project/assets/database/locales/global"

BIN
api/bun.lockb
View File

Binary file not shown.

3
api/package.json
View File

@ -12,7 +12,8 @@
"@elysiajs/swagger": "^1.1.6",
"axios": "^1.7.9",
"elysia": "^1.1.26",
"logestic": "^1.2.4"
"logestic": "^1.2.4",
"simple-git": "^3.27.0"
},
"devDependencies": {
"@types/bun": "latest"

157
api/src/services/CacheService.ts
View File

@ -1,73 +1,134 @@
import axios from "axios";
import { ILocalesCache, IHandbookCache, ICache, CacheKeys, CacheValues, ILocaleCache } from "@src/interfaces/ICaches";
import { IGiteaContentsResponse } from "@src/interfaces/IGiteaContentsResponse";
import { IGiteaItem } from "@src/interfaces/IGiteaItem";
import { IGiteaLocale } from "@src/interfaces/IGiteaLocale";
import { IGiteaHandbook } from "@src/interfaces/IGiteaHandbook";
import { promisify } from "util";
import simpleGit, { SimpleGit } from "simple-git";
import fs from "fs/promises";
import path from "path";
export class CacheService {
private static cache: Map<CacheKeys, CacheValues> = new Map();
private static repoPath = path.resolve("cache-repo");
private static async refreshItemsCache(): Promise<void> {
const items = (await axios.get<Record<string, IGiteaItem>>(Bun.env.GITEA_ITEMS_URL)).data;
const customization = (await axios.get<Record<string, IGiteaItem>>(Bun.env.GITEA_CUSTOMIZATION_URL)).data;
// Pull or clone the repository
private static async pullRepository(): Promise<void> {
const git: SimpleGit = simpleGit();
this.setCache("items", {
...items,
...customization
});
if (!(await fs.stat(this.repoPath).catch(() => false))) {
console.log("[CACHE] Cloning repository with sparse checkout...");
await git.clone(<string>Bun.env.GIT_REPO_URL, this.repoPath, ["--depth", "1", "--filter=blob:none"]);
// Enable sparse checkout
console.log("[CACHE] Initializing sparse checkout...");
await git.cwd(this.repoPath).raw(["sparse-checkout", "init", "--cone"]);
// Define the directories and specific files to include
const sparsePaths = [
<string>Bun.env.TEMPLATES_PATH, // Includes items.json and customization.json
<string>Bun.env.LOCALES_PATH, // Includes all locales files
];
console.log(`[CACHE] Setting sparse-checkout paths: ${sparsePaths.join(", ")}`);
await git.cwd(this.repoPath).raw(["sparse-checkout", "set", ...sparsePaths]);
} else {
console.log("[CACHE] Pulling updates with sparse checkout...");
await git.cwd(this.repoPath).pull();
}
console.log("[CACHE] Fetching specific LFS-managed files...");
try {
// Fetch and pull only the required LFS-managed file(s)
await git.cwd(this.repoPath).raw(["lfs", "fetch", "--include", <string>Bun.env.ITEMS_PATH]);
await git.cwd(this.repoPath).raw(["lfs", "pull", "--include", <string>Bun.env.ITEMS_PATH]);
} catch (error) {
console.error("[CACHE] Error fetching LFS data:", error);
}
}
// Refresh Items Cache
private static async refreshItemsCache(): Promise<void> {
const itemsPath = path.join(this.repoPath, <string>Bun.env.ITEMS_PATH);
const customizationPath = path.join(this.repoPath, <string>Bun.env.CUSTOMIZATION_PATH);
console.log(`[CACHE] Loading items from ${itemsPath}`);
console.log(`[CACHE] Loading customization from ${customizationPath}`);
try {
const items = JSON.parse(await fs.readFile(itemsPath, "utf-8"));
const customization = JSON.parse(await fs.readFile(customizationPath, "utf-8"));
this.setCache("items", { ...items, ...customization });
} catch (error) {
console.error("[CACHE] Error refreshing items cache:", error);
}
}
// Refresh Locales Cache
private static async refreshLocalesCache(): Promise<void> {
let locales: ILocalesCache = {};
const localesDir = path.join(this.repoPath, <string>Bun.env.LOCALES_PATH);
console.log(`[CACHE] Scanning locales directory: ${localesDir}`);
const localesList = (await axios.get<IGiteaContentsResponse[]>(Bun.env.GITEA_LOCALES_API_URL)).data;
try {
const localeFiles = await fs.readdir(localesDir);
let locales: ILocalesCache = {};
for (const locale of localesList) {
const localeName = locale.name.split(".json")[0].trim();
if (localeName.length === 0) {
continue;
}
const localeJson = (await axios.get<IGiteaLocale>(locale.download_url)).data;
const templates: ILocaleCache = {};
for (const item of Object.values(this.getCache("items"))) {
if (item._type === "Node" || localeJson[`${item._id} Name`] === undefined) {
for (const fileName of localeFiles) {
if (!fileName.endsWith(".json")) {
console.log(`[CACHE] Skipping non-JSON file: ${fileName}`);
continue;
}
templates[item._id] = {
"Name": localeJson[`${item._id} Name`],
"ShortName": localeJson[`${item._id} ShortName`],
"Description": localeJson[`${item._id} Description`]
const localeName = fileName.split(".json")[0].trim(); // Extract locale name from file name
console.log(`[CACHE] Processing locale file: ${fileName} (${localeName})`);
const localeJson = JSON.parse(await fs.readFile(path.join(localesDir, fileName), "utf-8"));
const templates: ILocaleCache = {};
for (const item of Object.values(this.getCache("items"))) {
if (item._type === "Node" || localeJson[`${item._id} Name`] === undefined) {
continue;
}
templates[item._id] = {
Name: localeJson[`${item._id} Name`],
ShortName: localeJson[`${item._id} ShortName`],
Description: localeJson[`${item._id} Description`],
};
}
locales[localeName] = templates;
}
this.setCache("locales", locales);
console.log(`[CACHE] Locales cache refreshed successfully.`);
} catch (error) {
console.error("[CACHE] Error refreshing locales cache:", error);
}
}
// Refresh Handbook Cache
private static async refreshHandbookCache(): Promise<void> {
const handbookPath = path.join(this.repoPath, <string>Bun.env.HANDBOOK_PATH);
console.log(`[CACHE] Loading handbook from ${handbookPath}`);
try {
const giteaHandbook = JSON.parse(await fs.readFile(handbookPath, "utf-8"));
const handbook: IHandbookCache = {};
for (const handbookItem of giteaHandbook.Items) {
handbook[handbookItem.Id] = {
ParentId: handbookItem.ParentId,
Price: handbookItem.Price,
};
}
locales[localeName] = templates;
this.setCache("handbook", handbook);
} catch (error) {
console.error("[CACHE] Error refreshing handbook cache:", error);
}
this.setCache("locales", locales);
}
private static async refreshHandbookCache(): Promise<void> {
let handbook: IHandbookCache = {};
const gitea_handbook = (await axios.get<{ Items: IGiteaHandbook[] }>(Bun.env.GITEA_HANDBOOK_URL)).data;
for (const handbookItem of gitea_handbook.Items) {
handbook[handbookItem.Id] = {
ParentId: handbookItem.ParentId,
Price: handbookItem.Price
};
}
this.setCache("handbook", handbook);
}
// Refresh All Caches
public static async refreshAllCache(): Promise<void> {
console.log("[CACHE] Refreshing cache...");
await this.pullRepository();
await this.refreshItemsCache();
await this.refreshLocalesCache();
await this.refreshHandbookCache();