mirror of
https://github.com/sp-tarkov/server.git
synced 2025-02-13 09:50:43 -05:00
Various Async improvements (#1044)
- Done some slight refactoring to `DatabaseImporter` to get rid of the old loading methods that have been sitting unused for sometime, as well as slightly refactoring `loadAsync` for better readability and using map's wherever possible, this should also yield a slight performance improvement? - Updated VFS to use node:fs/promises rather than our own promisfying of those methods. - Got rid of commands on VFS, I don't see why these are necessary anymore? If there's a good reason to still leave these I can revert this. - Changed loadImages to loadImagesAsync --------- Co-authored-by: Chomp <27521899+chompDev@users.noreply.github.com>
This commit is contained in:
GitHub
parent
2139c19a47
commit
2d7fdc0dc2
@ -743,7 +743,7 @@ export class SeasonalEventService {
|
||||
break;
|
||||
}
|
||||
|
||||
this.databaseImporter.loadImages(
|
||||
this.databaseImporter.loadImagesAsync(
|
||||
`${this.databaseImporter.getSptDataPath()}images/`,
|
||||
["traders"],
|
||||
["/files/trader/avatar/"],
|
||||
|
||||
@ -71,8 +71,8 @@ export class DatabaseImporter implements OnLoad {
|
||||
await this.hydrateDatabase(this.filepath);
|
||||
|
||||
const imageFilePath = `${this.filepath}images/`;
|
||||
const directories = this.vfs.getDirs(imageFilePath);
|
||||
this.loadImages(imageFilePath, directories, [
|
||||
const directories = await this.vfs.getDirsAsync(imageFilePath);
|
||||
await this.loadImagesAsync(imageFilePath, directories, [
|
||||
"/files/achievement/",
|
||||
"/files/CONTENT/banners/",
|
||||
"/files/handbook/",
|
||||
@ -142,10 +142,10 @@ export class DatabaseImporter implements OnLoad {
|
||||
* Find and map files with image router inside a designated path
|
||||
* @param filepath Path to find files in
|
||||
*/
|
||||
public loadImages(filepath: string, directories: string[], routes: string[]): void {
|
||||
public async loadImagesAsync(filepath: string, directories: string[], routes: string[]): Promise<void> {
|
||||
for (const directoryIndex in directories) {
|
||||
// Get all files in directory
|
||||
const filesInDirectory = this.vfs.getFiles(`${filepath}${directories[directoryIndex]}`);
|
||||
const filesInDirectory = await this.vfs.getFilesAsync(`${filepath}${directories[directoryIndex]}`);
|
||||
for (const file of filesInDirectory) {
|
||||
// Register each file in image router
|
||||
const filename = this.vfs.stripExtension(file);
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import { JsonUtil } from "@spt/utils/JsonUtil";
|
||||
import { ProgressWriter } from "@spt/utils/ProgressWriter";
|
||||
import { VFS } from "@spt/utils/VFS";
|
||||
import { Queue } from "@spt/utils/collections/queue/Queue";
|
||||
import { inject, injectable } from "tsyringe";
|
||||
|
||||
@injectable()
|
||||
@ -11,141 +10,72 @@ export class ImporterUtil {
|
||||
@inject("JsonUtil") protected jsonUtil: JsonUtil,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Load files into js objects recursively (asynchronous)
|
||||
* @param filepath Path to folder with files
|
||||
* @returns Promise<T> return T type associated with this class
|
||||
*/
|
||||
public async loadRecursiveAsync<T>(
|
||||
filepath: string,
|
||||
onReadCallback: (fileWithPath: string, data: string) => void = () => {},
|
||||
onObjectDeserialized: (fileWithPath: string, object: any) => void = () => {},
|
||||
): Promise<T> {
|
||||
const result = {} as T;
|
||||
|
||||
// get all filepaths
|
||||
const files = this.vfs.getFiles(filepath);
|
||||
const directories = this.vfs.getDirs(filepath);
|
||||
|
||||
// add file content to result
|
||||
for (const file of files) {
|
||||
if (this.vfs.getFileExtension(file) === "json") {
|
||||
const filename = this.vfs.stripExtension(file);
|
||||
const filePathAndName = `${filepath}${file}`;
|
||||
await this.vfs.readFileAsync(filePathAndName).then((fileData) => {
|
||||
onReadCallback(filePathAndName, fileData);
|
||||
const fileDeserialized = this.jsonUtil.deserializeWithCacheCheck(fileData, filePathAndName);
|
||||
onObjectDeserialized(filePathAndName, fileDeserialized);
|
||||
result[filename] = fileDeserialized;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// deep tree search
|
||||
for (const dir of directories) {
|
||||
result[dir] = this.loadRecursiveAsync(`${filepath}${dir}/`);
|
||||
}
|
||||
|
||||
// set all loadRecursive to be executed asynchronously
|
||||
const resEntries = Object.entries(result);
|
||||
const resResolved = await Promise.all(resEntries.map((ent) => ent[1]));
|
||||
for (let resIdx = 0; resIdx < resResolved.length; resIdx++) {
|
||||
resEntries[resIdx][1] = resResolved[resIdx];
|
||||
}
|
||||
|
||||
// return the result of all async fetch
|
||||
return Object.fromEntries(resEntries) as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load files into js objects recursively (synchronous)
|
||||
* @param filepath Path to folder with files
|
||||
* @returns
|
||||
*/
|
||||
public loadRecursive<T>(
|
||||
filepath: string,
|
||||
onReadCallback: (fileWithPath: string, data: string) => void = () => {},
|
||||
onObjectDeserialized: (fileWithPath: string, object: any) => void = () => {},
|
||||
): T {
|
||||
const result = {} as T;
|
||||
|
||||
// get all filepaths
|
||||
const files = this.vfs.getFiles(filepath);
|
||||
const directories = this.vfs.getDirs(filepath);
|
||||
|
||||
// add file content to result
|
||||
for (const file of files) {
|
||||
if (this.vfs.getFileExtension(file) === "json") {
|
||||
const filename = this.vfs.stripExtension(file);
|
||||
const filePathAndName = `${filepath}${file}`;
|
||||
const fileData = this.vfs.readFile(filePathAndName);
|
||||
onReadCallback(filePathAndName, fileData);
|
||||
const fileDeserialized = this.jsonUtil.deserializeWithCacheCheck(fileData, filePathAndName);
|
||||
onObjectDeserialized(filePathAndName, fileDeserialized);
|
||||
result[filename] = fileDeserialized;
|
||||
}
|
||||
}
|
||||
|
||||
// deep tree search
|
||||
for (const dir of directories) {
|
||||
result[dir] = this.loadRecursive(`${filepath}${dir}/`);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async loadAsync<T>(
|
||||
filepath: string,
|
||||
strippablePath = "",
|
||||
onReadCallback: (fileWithPath: string, data: string) => void = () => {},
|
||||
onObjectDeserialized: (fileWithPath: string, object: any) => void = () => {},
|
||||
): Promise<T> {
|
||||
const directoriesToRead = new Queue<string>();
|
||||
const filesToProcess = new Queue<VisitNode>();
|
||||
|
||||
const promises = new Array<Promise<any>>();
|
||||
|
||||
const result = {} as T;
|
||||
|
||||
const files = this.vfs.getFiles(filepath);
|
||||
const directories = this.vfs.getDirs(filepath);
|
||||
// Fetch files and directories concurrently for the root path
|
||||
const [files, directories] = await Promise.all([
|
||||
this.vfs.getFilesAsync(filepath),
|
||||
this.vfs.getDirsAsync(filepath),
|
||||
]);
|
||||
|
||||
directoriesToRead.enqueueAll(directories.map((d) => `${filepath}${d}`));
|
||||
filesToProcess.enqueueAll(files.map((f) => new VisitNode(filepath, f)));
|
||||
// Queue to process files and directories for the root path first.
|
||||
const filesToProcess = files.map((f) => new VisitNode(filepath, f));
|
||||
const directoriesToRead = directories.map((d) => `${filepath}${d}`);
|
||||
|
||||
while (directoriesToRead.length !== 0) {
|
||||
const directory = directoriesToRead.dequeue();
|
||||
if (!directory) continue;
|
||||
filesToProcess.enqueueAll(this.vfs.getFiles(directory).map((f) => new VisitNode(`${directory}/`, f)));
|
||||
directoriesToRead.enqueueAll(this.vfs.getDirs(directory).map((d) => `${directory}/${d}`));
|
||||
const allFiles = [...filesToProcess];
|
||||
|
||||
// Method to traverse directories and collect all files recursively
|
||||
const traverseDirectories = async (directory: string) => {
|
||||
const [directoryFiles, subDirectories] = await Promise.all([
|
||||
this.vfs.getFilesAsync(directory),
|
||||
this.vfs.getDirsAsync(directory),
|
||||
]);
|
||||
|
||||
// Add the files from this directory to the processing queue
|
||||
const fileNodes = directoryFiles.map((f) => new VisitNode(directory, f));
|
||||
allFiles.push(...fileNodes);
|
||||
|
||||
// Recurse into subdirectories
|
||||
for (const subDirectory of subDirectories) {
|
||||
await traverseDirectories(`${directory}/${subDirectory}`);
|
||||
}
|
||||
};
|
||||
|
||||
// Start recursive directory traversal
|
||||
const traversalPromises = directoriesToRead.map((dir) => traverseDirectories(dir));
|
||||
await Promise.all(traversalPromises); // Ensure all directories are processed
|
||||
|
||||
// Setup the progress writer with the total amount of files to load
|
||||
const progressWriter = new ProgressWriter(allFiles.length);
|
||||
|
||||
const fileProcessingPromises = allFiles.map(async (fileNode) => {
|
||||
if (this.vfs.getFileExtension(fileNode.fileName) !== "json") {
|
||||
return Promise.resolve(); // Skip non-JSON files
|
||||
}
|
||||
|
||||
const progressWriter = new ProgressWriter(filesToProcess.length - 1);
|
||||
// Ensure we're attempting to read the correct file path
|
||||
const filePathAndName = `${fileNode.filePath}${fileNode.filePath.endsWith("/") ? "" : "/"}${fileNode.fileName}`;
|
||||
|
||||
while (filesToProcess.length !== 0) {
|
||||
const fileNode = filesToProcess.dequeue();
|
||||
if (!fileNode || this.vfs.getFileExtension(fileNode.fileName) !== "json") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const filePathAndName = `${fileNode.filePath}${fileNode.fileName}`;
|
||||
promises.push(
|
||||
this.vfs
|
||||
.readFileAsync(filePathAndName)
|
||||
.then(async (fileData) => {
|
||||
try {
|
||||
const fileData = await this.vfs.readFileAsync(filePathAndName);
|
||||
onReadCallback(filePathAndName, fileData);
|
||||
return this.jsonUtil.deserializeWithCacheCheckAsync<any>(fileData, filePathAndName);
|
||||
})
|
||||
.then(async (fileDeserialized) => {
|
||||
const fileDeserialized = await this.jsonUtil.deserializeWithCacheCheckAsync<any>(fileData, filePathAndName);
|
||||
onObjectDeserialized(filePathAndName, fileDeserialized);
|
||||
const strippedFilePath = this.vfs.stripExtension(filePathAndName).replace(filepath, "");
|
||||
this.placeObject(fileDeserialized, strippedFilePath, result, strippablePath);
|
||||
})
|
||||
.then(() => progressWriter.increment()),
|
||||
);
|
||||
} finally {
|
||||
return progressWriter.increment(); // Update progress after each file
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(promises).catch((e) => console.error(e));
|
||||
// Wait for all file processing to complete
|
||||
await Promise.all(fileProcessingPromises).catch((e) => console.error(e));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -2,8 +2,8 @@ import "reflect-metadata";
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import * as fsPromises from "node:fs/promises";
|
||||
import path, { resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import type { IAsyncQueue } from "@spt/models/spt/utils/IAsyncQueue";
|
||||
import { writeFileSync } from "atomically";
|
||||
import { checkSync, lockSync, unlockSync } from "proper-lockfile";
|
||||
@ -11,37 +11,7 @@ import { inject, injectable } from "tsyringe";
|
||||
|
||||
@injectable()
|
||||
export class VFS {
|
||||
accessFilePromisify: (path: fs.PathLike, mode?: number) => Promise<void>;
|
||||
copyFilePromisify: (src: fs.PathLike, dst: fs.PathLike, flags?: number) => Promise<void>;
|
||||
mkdirPromisify: (
|
||||
path: fs.PathLike,
|
||||
options: fs.MakeDirectoryOptions & { recursive: true },
|
||||
) => Promise<string | undefined>;
|
||||
|
||||
readFilePromisify: (path: fs.PathLike) => Promise<Buffer>;
|
||||
writeFilePromisify: (path: fs.PathLike, data: string, options?: any) => Promise<void>;
|
||||
readdirPromisify: (
|
||||
path: fs.PathLike,
|
||||
options?: BufferEncoding | { encoding: BufferEncoding; withFileTypes?: false },
|
||||
) => Promise<string[]>;
|
||||
|
||||
statPromisify: (path: fs.PathLike, options?: fs.StatOptions & { bigint?: false }) => Promise<fs.Stats>;
|
||||
unlinkPromisify: (path: fs.PathLike) => Promise<void>;
|
||||
rmdirPromisify: (path: fs.PathLike) => Promise<void>;
|
||||
renamePromisify: (oldPath: fs.PathLike, newPath: fs.PathLike) => Promise<void>;
|
||||
|
||||
constructor(@inject("AsyncQueue") protected asyncQueue: IAsyncQueue) {
|
||||
this.accessFilePromisify = promisify(fs.access);
|
||||
this.copyFilePromisify = promisify(fs.copyFile);
|
||||
this.mkdirPromisify = promisify(fs.mkdir);
|
||||
this.readFilePromisify = promisify(fs.readFile);
|
||||
this.writeFilePromisify = promisify(fs.writeFile);
|
||||
this.readdirPromisify = promisify(fs.readdir);
|
||||
this.statPromisify = promisify(fs.stat);
|
||||
this.unlinkPromisify = promisify(fs.unlinkSync);
|
||||
this.rmdirPromisify = promisify(fs.rmdir);
|
||||
this.renamePromisify = promisify(fs.renameSync);
|
||||
}
|
||||
constructor(@inject("AsyncQueue") protected asyncQueue: IAsyncQueue) {}
|
||||
|
||||
public exists(filepath: fs.PathLike): boolean {
|
||||
return fs.existsSync(filepath);
|
||||
@ -49,10 +19,7 @@ export class VFS {
|
||||
|
||||
public async existsAsync(filepath: fs.PathLike): Promise<boolean> {
|
||||
try {
|
||||
// Create the command to add to the queue
|
||||
const command = { uuid: crypto.randomUUID(), cmd: async () => await this.accessFilePromisify(filepath) };
|
||||
// Wait for the command completion
|
||||
await this.asyncQueue.waitFor(command);
|
||||
await fsPromises.access(filepath);
|
||||
|
||||
// If no Exception, the file exists
|
||||
return true;
|
||||
@ -67,8 +34,7 @@ export class VFS {
|
||||
}
|
||||
|
||||
public async copyAsync(filepath: fs.PathLike, target: fs.PathLike): Promise<void> {
|
||||
const command = { uuid: crypto.randomUUID(), cmd: async () => await this.copyFilePromisify(filepath, target) };
|
||||
await this.asyncQueue.waitFor(command);
|
||||
await fsPromises.copyFile(filepath, target);
|
||||
}
|
||||
|
||||
public createDir(filepath: string): void {
|
||||
@ -76,12 +42,7 @@ export class VFS {
|
||||
}
|
||||
|
||||
public async createDirAsync(filepath: string): Promise<void> {
|
||||
const command = {
|
||||
uuid: crypto.randomUUID(),
|
||||
cmd: async () =>
|
||||
await this.mkdirPromisify(filepath.substr(0, filepath.lastIndexOf("/")), { recursive: true }),
|
||||
};
|
||||
await this.asyncQueue.waitFor(command);
|
||||
await fsPromises.mkdir(filepath.slice(0, filepath.lastIndexOf("/")), { recursive: true });
|
||||
}
|
||||
|
||||
public copyDir(filepath: string, target: string, fileExtensions?: string | string[]): void {
|
||||
@ -133,18 +94,18 @@ export class VFS {
|
||||
}
|
||||
|
||||
public async readFileAsync(path: fs.PathLike): Promise<string> {
|
||||
const read = await this.readFilePromisify(path);
|
||||
const read = await fsPromises.readFile(path);
|
||||
if (this.isBuffer(read)) {
|
||||
return read.toString();
|
||||
}
|
||||
return read;
|
||||
}
|
||||
|
||||
private isBuffer(value: any): value is Buffer {
|
||||
return value?.write && value.toString && value.toJSON && value.equals;
|
||||
private isBuffer(value: Buffer | string): value is Buffer {
|
||||
return Buffer.isBuffer(value);
|
||||
}
|
||||
|
||||
public writeFile(filepath: any, data = "", append = false, atomic = true): void {
|
||||
public writeFile(filepath: string, data = "", append = false, atomic = true): void {
|
||||
const options = append ? { flag: "a" } : { flag: "w" };
|
||||
|
||||
if (!this.exists(filepath)) {
|
||||
@ -163,18 +124,18 @@ export class VFS {
|
||||
releaseCallback();
|
||||
}
|
||||
|
||||
public async writeFileAsync(filepath: any, data = "", append = false, atomic = true): Promise<void> {
|
||||
public async writeFileAsync(filepath: string, data = "", append = false, atomic = true): Promise<void> {
|
||||
const options = append ? { flag: "a" } : { flag: "w" };
|
||||
|
||||
if (!(await this.exists(filepath))) {
|
||||
await this.createDir(filepath);
|
||||
await this.writeFilePromisify(filepath, "");
|
||||
if (!(await this.existsAsync(filepath))) {
|
||||
await this.createDirAsync(filepath);
|
||||
await fsPromises.writeFile(filepath, "");
|
||||
}
|
||||
|
||||
if (!append && atomic) {
|
||||
await this.writeFilePromisify(filepath, data);
|
||||
await fsPromises.writeFile(filepath, data);
|
||||
} else {
|
||||
await this.writeFilePromisify(filepath, data, options);
|
||||
await fsPromises.writeFile(filepath, data, options);
|
||||
}
|
||||
}
|
||||
|
||||
@ -185,11 +146,8 @@ export class VFS {
|
||||
}
|
||||
|
||||
public async getFilesAsync(filepath: string): Promise<string[]> {
|
||||
const addr = await this.readdirPromisify(filepath);
|
||||
return addr.filter(async (item) => {
|
||||
const stat = await this.statPromisify(path.join(filepath, item));
|
||||
return stat.isFile();
|
||||
});
|
||||
const entries = await fsPromises.readdir(filepath, { withFileTypes: true });
|
||||
return entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
|
||||
}
|
||||
|
||||
public getDirs(filepath: string): string[] {
|
||||
@ -199,11 +157,8 @@ export class VFS {
|
||||
}
|
||||
|
||||
public async getDirsAsync(filepath: string): Promise<string[]> {
|
||||
const addr = await this.readdirPromisify(filepath);
|
||||
return addr.filter(async (item) => {
|
||||
const stat = await this.statPromisify(path.join(filepath, item));
|
||||
return stat.isDirectory();
|
||||
});
|
||||
const entries = await fsPromises.readdir(filepath, { withFileTypes: true });
|
||||
return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
|
||||
}
|
||||
|
||||
public removeFile(filepath: string): void {
|
||||
@ -211,7 +166,7 @@ export class VFS {
|
||||
}
|
||||
|
||||
public async removeFileAsync(filepath: string): Promise<void> {
|
||||
await this.unlinkPromisify(filepath);
|
||||
await fsPromises.unlink(filepath);
|
||||
}
|
||||
|
||||
public removeDir(filepath: string): void {
|
||||
@ -244,7 +199,7 @@ export class VFS {
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
await this.rmdirPromisify(filepath);
|
||||
await fsPromises.rmdir(filepath);
|
||||
}
|
||||
|
||||
public rename(oldPath: string, newPath: string): void {
|
||||
@ -252,18 +207,18 @@ export class VFS {
|
||||
}
|
||||
|
||||
public async renameAsync(oldPath: string, newPath: string): Promise<void> {
|
||||
await this.renamePromisify(oldPath, newPath);
|
||||
await fsPromises.rename(oldPath, newPath);
|
||||
}
|
||||
|
||||
protected lockFileSync(filepath: any): () => void {
|
||||
protected lockFileSync(filepath: string): () => void {
|
||||
return lockSync(filepath);
|
||||
}
|
||||
|
||||
protected checkFileSync(filepath: any): boolean {
|
||||
protected checkFileSync(filepath: string): boolean {
|
||||
return checkSync(filepath);
|
||||
}
|
||||
|
||||
protected unlockFileSync(filepath: any): void {
|
||||
protected unlockFileSync(filepath: string): void {
|
||||
unlockSync(filepath);
|
||||
}
|
||||
|
||||
|
||||
@ -1,43 +0,0 @@
|
||||
import { LinkedList } from "@spt/utils/collections/lists/LinkedList";
|
||||
|
||||
export class Queue<T> {
|
||||
private list: LinkedList<T>;
|
||||
|
||||
public get length(): number {
|
||||
return this.list.length;
|
||||
}
|
||||
|
||||
constructor() {
|
||||
this.list = new LinkedList<T>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds an element to the end of the queue.
|
||||
*/
|
||||
public enqueue(element: T): void {
|
||||
this.list.append(element);
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterates over the elements received and adds each one to the end of the queue.
|
||||
*/
|
||||
public enqueueAll(elements: T[]): void {
|
||||
for (const element of elements) {
|
||||
this.enqueue(element);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the first element from the queue and returns it's value. If the queue is empty, undefined is returned and the queue is not modified.
|
||||
*/
|
||||
public dequeue(): T | undefined {
|
||||
return this.list.shift();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the first element's value.
|
||||
*/
|
||||
public peek(): T | undefined {
|
||||
return this.list.getHead();
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user