spt/forge
spt/forge
0
0
Fork 0
mirror of https://github.com/sp-tarkov/forge.git synced 2025-02-12 12:10:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

Converts import command to a queued job

Browse Source
This commit is contained in:
Refringe 2024-06-12 16:52:32 -04:00
parent 8d1242ed55
commit be0eecf1e9
Signed by: Refringe
SSH Key Fingerprint: SHA256:t865XsQpfTeqPRBMN2G6+N8wlDjkgUCZF3WGW6O9N/k
9 changed files with 592 additions and 497 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

478
app/Console/Commands/ImportHub.php
View File

@ -2,20 +2,8 @@
namespace App\Console\Commands;
use App\Models\License;
use App\Models\Mod;
use App\Models\ModVersion;
use App\Models\SptVersion;
use App\Models\User;
use Carbon\Carbon;
use App\Jobs\ImportHubData;
use Illuminate\Console\Command;
use Illuminate\Support\Benchmark;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use League\HTMLToMarkdown\HtmlConverter;
use Stevebauman\Purify\Facades\Purify;
class ImportHub extends Command
{
@ -23,469 +11,11 @@ class ImportHub extends Command
protected $description = 'Connects to the Hub database and imports the data into the Laravel database.';
/**
* Execute the console command.
*/
public function handle(): void
{
// This may take a minute or two...
set_time_limit(0);
// Add the ImportHubData job to the queue.
ImportHubData::dispatch()->onQueue('long');
$this->newLine();
$totalTime = Benchmark::value(function () {
$loadDataTime = Benchmark::value(function () {
$this->loadData();
});
$this->info('Execution time: '.round($loadDataTime[1], 2).'ms');
$this->newLine();
$importUsersTime = Benchmark::value(function () {
$this->importUsers();
});
$this->info('Execution time: '.round($importUsersTime[1], 2).'ms');
$this->newLine();
$importLicensesTime = Benchmark::value(function () {
$this->importLicenses();
});
$this->info('Execution time: '.round($importLicensesTime[1], 2).'ms');
$this->newLine();
$importSptVersionsTime = Benchmark::value(function () {
$this->importSptVersions();
});
$this->info('Execution time: '.round($importSptVersionsTime[1], 2).'ms');
$this->newLine();
$importModsTime = Benchmark::value(function () {
$this->importMods();
});
$this->info('Execution time: '.round($importModsTime[1], 2).'ms');
$this->newLine();
$importModVersionsTime = Benchmark::value(function () {
$this->importModVersions();
});
$this->info('Execution time: '.round($importModVersionsTime[1], 2).'ms');
$this->newLine();
});
// Disconnect from the Hub database, clearing temporary tables.
DB::connection('mysql_hub')->disconnect();
$this->newLine();
$this->info('Data imported successfully');
$this->info('Total execution time: '.round($totalTime[1], 2).'ms');
$this->newLine();
$this->info('Refreshing Meilisearch indexes...');
$this->call('scout:delete-all-indexes');
$this->call('scout:sync-index-settings');
$this->call('scout:import', ['model' => '\App\Models\Mod']);
$this->newLine();
$this->info('Done');
}
protected function loadData(): void
{
// We're just going to dump a few things in memory to escape the N+1 problem.
$this->output->write('Loading data into memory... ');
$this->bringFileOptionsLocal();
$this->bringFileContentLocal();
$this->bringFileVersionLabelsLocal();
$this->bringFileVersionContentLocal();
$this->info('Done.');
}
protected function importUsers(): void
{
$totalInserted = 0;
foreach (DB::connection('mysql_hub')->table('wcf1_user')->orderBy('userID')->cursor() as $wolt) {
$registrationDate = Carbon::parse($wolt->registrationDate, 'UTC');
if ($registrationDate->isFuture()) {
$registrationDate = now('UTC');
}
$registrationDate->setTimezone('UTC');
$insertData = [
'hub_id' => $wolt->userID,
'name' => $wolt->username,
'email' => mb_convert_case($wolt->email, MB_CASE_LOWER, 'UTF-8'),
'password' => $this->cleanPasswordHash($wolt->password),
'created_at' => $registrationDate,
'updated_at' => now('UTC')->toDateTimeString(),
];
User::upsert($insertData, ['hub_id'], ['name', 'email', 'password', 'created_at', 'updated_at']);
$totalInserted++;
// Log every 2500 users processed
if ($totalInserted % 2500 == 0) {
$this->line('Processed 2500 users. Total processed so far: '.$totalInserted);
}
}
$this->info('Total users processed: '.$totalInserted);
}
protected function cleanPasswordHash(string $password): string
{
// The hub passwords are hashed sometimes with a prefix of the hash type. We only want the hash.
// If it's not Bcrypt, they'll have to reset their password. Tough luck.
return str_replace(['Bcrypt:', 'cryptMD5:', 'cryptMD5::'], '', $password);
}
protected function importLicenses(): void
{
$totalInserted = 0;
DB::connection('mysql_hub')
->table('filebase1_license')
->chunkById(100, function (Collection $licenses) use (&$totalInserted) {
$insertData = [];
foreach ($licenses as $license) {
$insertData[] = [
'hub_id' => $license->licenseID,
'name' => $license->licenseName,
'link' => $license->licenseURL,
];
}
if (! empty($insertData)) {
DB::table('licenses')->upsert($insertData, ['hub_id'], ['name', 'link']);
$totalInserted += count($insertData);
$this->line('Processed '.count($insertData).' licenses. Total processed so far: '.$totalInserted);
}
unset($insertData);
unset($licenses);
}, 'licenseID');
$this->info('Total licenses processed: '.$totalInserted);
}
protected function importSptVersions(): void
{
$totalInserted = 0;
DB::connection('mysql_hub')
->table('wcf1_label')
->where('groupID', 1)
->chunkById(100, function (Collection $versions) use (&$totalInserted) {
$insertData = [];
foreach ($versions as $version) {
$insertData[] = [
'hub_id' => $version->labelID,
'version' => $version->label,
'color_class' => $this->translateColour($version->cssClassName),
];
}
if (! empty($insertData)) {
DB::table('spt_versions')->upsert($insertData, ['hub_id'], ['version', 'color_class']);
$totalInserted += count($insertData);
$this->line('Processed '.count($insertData).' SPT Versions. Total processed so far: '.$totalInserted);
}
unset($insertData);
unset($versions);
}, 'labelID');
$this->info('Total licenses processed: '.$totalInserted);
}
protected function translateColour(string $colour = ''): string
{
return match ($colour) {
'green' => 'green',
'slightly-outdated' => 'lime',
'yellow' => 'yellow',
'red' => 'red',
default => 'gray',
};
}
protected function importMods(): void
{
$command = $this;
$totalInserted = 0;
$curl = curl_init();
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
DB::connection('mysql_hub')
->table('filebase1_file')
->chunkById(100, function (Collection $mods) use (&$command, &$curl, &$totalInserted) {
foreach ($mods as $mod) {
$modContent = DB::table('temp_file_content')
->where('fileID', $mod->fileID)
->orderBy('fileID', 'desc')
->first();
$optionSourceCode = DB::table('temp_file_option_values')
->select('optionValue as source_code_link')
->where('fileID', $mod->fileID)
->whereIn('optionID', [5, 1])
->whereNot('optionValue', '')
->orderByDesc('optionID')
->first();
$optionContainsAi = DB::table('temp_file_option_values')
->select('optionValue as contains_ai')
->where('fileID', $mod->fileID)
->where('optionID', 7)
->whereNot('optionValue', '')
->first();
$optionContainsAds = DB::table('temp_file_option_values')
->select('optionValue as contains_ads')
->where('fileID', $mod->fileID)
->where('optionID', 3)
->whereNot('optionValue', '')
->first();
$versionLabel = DB::table('temp_file_version_labels')
->select('labelID')
->where('objectID', $mod->fileID)
->orderBy('labelID', 'desc')
->first();
if (empty($versionLabel)) {
continue;
}
$insertData[] = [
'hub_id' => (int) $mod->fileID,
'user_id' => User::whereHubId($mod->userID)->value('id'),
'name' => $modContent?->subject ?? '',
'slug' => Str::slug($modContent?->subject) ?? '',
'teaser' => Str::limit($modContent?->teaser) ?? '',
'description' => $this->convertModDescription($modContent?->message ?? ''),
'thumbnail' => $this->fetchModThumbnail($command, $curl, $mod->fileID, $mod->iconHash, $mod->iconExtension),
'license_id' => License::whereHubId($mod->licenseID)->value('id'),
'source_code_link' => $optionSourceCode?->source_code_link ?? '',
'featured' => (bool) $mod->isFeatured,
'contains_ai_content' => (bool) $optionContainsAi?->contains_ai ?? false,
'contains_ads' => (bool) $optionContainsAds?->contains_ads ?? false,
'disabled' => (bool) $mod->isDisabled,
'created_at' => Carbon::parse($mod->time, 'UTC'),
'updated_at' => Carbon::parse($mod->lastChangeTime, 'UTC'),
];
}
if (! empty($insertData)) {
Mod::upsert($insertData, ['hub_id'], ['user_id', 'name', 'slug', 'teaser', 'description', 'thumbnail', 'license_id', 'source_code_link', 'featured', 'contains_ai_content', 'disabled', 'created_at', 'updated_at']);
$totalInserted += count($insertData);
$command->line('Processed '.count($insertData).' mods. Total processed so far: '.$totalInserted);
}
unset($insertData);
unset($mods);
}, 'fileID');
curl_close($curl);
$this->info('Total mods processed: '.$totalInserted);
}
protected function bringFileOptionsLocal(): void
{
DB::statement('CREATE TEMPORARY TABLE temp_file_option_values (
fileID INT,
optionID INT,
optionValue VARCHAR(255)
)');
DB::connection('mysql_hub')
->table('filebase1_file_option_value')
->orderBy('fileID')
->chunk(200, function ($options) {
foreach ($options as $option) {
DB::table('temp_file_option_values')->insert([
'fileID' => $option->fileID,
'optionID' => $option->optionID,
'optionValue' => $option->optionValue,
]);
}
});
}
protected function bringFileContentLocal(): void
{
DB::statement('CREATE TEMPORARY TABLE temp_file_content (
fileID INT,
subject VARCHAR(255),
teaser VARCHAR(255),
message LONGTEXT
)');
DB::connection('mysql_hub')
->table('filebase1_file_content')
->orderBy('fileID')
->chunk(200, function ($contents) {
foreach ($contents as $content) {
DB::table('temp_file_content')->insert([
'fileID' => $content->fileID,
'subject' => $content->subject,
'teaser' => $content->teaser,
'message' => $content->message,
]);
}
});
}
protected function fetchModThumbnail($command, $curl, string $fileID, string $thumbnailHash, string $thumbnailExtension): string
{
if (empty($fileID) || empty($thumbnailHash) || empty($thumbnailExtension)) {
return '';
}
// Only the first two characters of the icon hash.
$hashShort = substr($thumbnailHash, 0, 2);
$hubUrl = "https://hub.sp-tarkov.com/files/images/file/$hashShort/$fileID.$thumbnailExtension";
$relativePath = "mods/$thumbnailHash.$thumbnailExtension";
$disk = match (config('app.env')) {
'production' => 'r2',
default => 'local',
};
// Check to make sure the image doesn't already exist.
if (Storage::disk($disk)->exists($relativePath)) {
return $relativePath;
}
$command->output->write("Downloading mod thumbnail: $hubUrl... ");
curl_setopt($curl, CURLOPT_URL, $hubUrl);
$image = curl_exec($curl);
if ($image === false) {
$command->error('Error: '.curl_error($curl));
} else {
Storage::disk($disk)->put($relativePath, $image);
$command->info('Done.');
}
return $relativePath;
}
protected function importModVersions(): void
{
$command = $this;
$totalInserted = 0;
DB::connection('mysql_hub')
->table('filebase1_file_version')
->chunkById(500, function (Collection $versions) use (&$command, &$totalInserted) {
foreach ($versions as $version) {
$versionContent = DB::table('temp_file_version_content')
->select('description')
->where('versionID', $version->versionID)
->orderBy('versionID', 'desc')
->first();
$optionVirusTotal = DB::table('temp_file_option_values')
->select('optionValue as virus_total_link')
->where('fileID', $version->fileID)
->whereIn('optionID', [6, 2])
->whereNot('optionValue', '')
->orderByDesc('optionID')
->first();
$versionLabel = DB::table('temp_file_version_labels')
->select('labelID')
->where('objectID', $version->fileID)
->orderBy('labelID', 'desc')
->first();
$modId = Mod::whereHubId($version->fileID)->value('id');
if (empty($versionLabel) || empty($modId)) {
continue;
}
$insertData[] = [
'hub_id' => $version->versionID,
'mod_id' => $modId,
'version' => $version->versionNumber,
'description' => $this->convertModDescription($versionContent->description ?? ''),
'link' => $version->downloadURL,
'spt_version_id' => SptVersion::whereHubId($versionLabel->labelID)->value('id'),
'virus_total_link' => $optionVirusTotal?->virus_total_link ?? '',
'downloads' => max((int) $version->downloads, 0), // Ensure the value is at least 0
'disabled' => (bool) $version->isDisabled,
'created_at' => Carbon::parse($version->uploadTime, 'UTC'),
'updated_at' => Carbon::parse($version->uploadTime, 'UTC'),
];
}
if (! empty($insertData)) {
ModVersion::upsert($insertData, ['hub_id'], ['mod_id', 'version', 'description', 'link', 'spt_version_id', 'virus_total_link', 'downloads', 'created_at', 'updated_at']);
$totalInserted += count($insertData);
$command->line('Processed '.count($insertData).' mod versions. Total processed so far: '.$totalInserted);
}
unset($insertData);
unset($version);
}, 'versionID');
$this->info('Total mod versions processed: '.$totalInserted);
}
protected function bringFileVersionLabelsLocal(): void
{
DB::statement('CREATE TEMPORARY TABLE temp_file_version_labels (
labelID INT,
objectID INT
)');
DB::connection('mysql_hub')
->table('wcf1_label_object')
->where('objectTypeID', 387)
->orderBy('labelID')
->chunk(200, function ($options) {
foreach ($options as $option) {
DB::table('temp_file_version_labels')->insert([
'labelID' => $option->labelID,
'objectID' => $option->objectID,
]);
}
});
}
protected function bringFileVersionContentLocal(): void
{
DB::statement('CREATE TEMPORARY TABLE temp_file_version_content (
versionID INT,
description TEXT
)');
DB::connection('mysql_hub')
->table('filebase1_file_version_content')
->orderBy('versionID')
->chunk(200, function ($options) {
foreach ($options as $option) {
DB::table('temp_file_version_content')->insert([
'versionID' => $option->versionID,
'description' => $option->description,
]);
}
});
}
protected function convertModDescription(string $description): string
{
// Alright, hear me out... Shut up.
$converter = new HtmlConverter();
return $converter->convert(Purify::clean($description));
$this->info('The import job has been added to the queue.');
}
}

15
app/Console/Commands/UploadAssets.php
View File

@ -8,24 +8,15 @@ use Illuminate\Support\Facades\Storage;
class UploadAssets extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:upload-assets';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Uploads the Vite build assets to Cloudflare R2';
/**
* Execute the console command.
* This command uploads the Vite build assets to Cloudflare R2. Typically, this will be run after the assets have
* been built and the application is ready to deploy from within the production environment build process.
*/
public function handle()
public function handle(): void
{
$this->info('Publishing assets...');

518
app/Jobs/ImportHubData.php Normal file
View File

@ -0,0 +1,518 @@
<?php
namespace App\Jobs;
use App\Models\License;
use App\Models\Mod;
use App\Models\ModVersion;
use App\Models\SptVersion;
use App\Models\User;
use Carbon\Carbon;
use CurlHandle;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use League\HTMLToMarkdown\HtmlConverter;
use Stevebauman\Purify\Facades\Purify;
class ImportHubData implements ShouldBeUnique, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function handle(): void
{
// Stream some data locally so that we don't have to keep accessing the Hub's database. Use MySQL temporary
// tables to store the data to save on memory; we don't want this to be a memory hog.
$this->bringFileOptionsLocal();
$this->bringFileContentLocal();
$this->bringFileVersionLabelsLocal();
$this->bringFileVersionContentLocal();
// Begin to import the data into the permanent local database tables.
$this->importUsers();
$this->importLicenses();
$this->importSptVersions();
$this->importMods();
$this->importModVersions();
// Ensure that we've disconnected from the Hub database, clearing temporary tables.
DB::connection('mysql_hub')->disconnect();
// Reindex the Meilisearch index.
Artisan::call('scout:delete-all-indexes');
Artisan::call('scout:sync-index-settings');
Artisan::call('scout:import', ['model' => '\App\Models\Mod']);
}
/**
* Bring the file options from the Hub database to the local database temporary table.
*/
protected function bringFileOptionsLocal(): void
{
if (Schema::hasTable('temp_file_option_values')) {
DB::statement('DROP TABLE temp_file_option_values');
}
DB::statement('CREATE TEMPORARY TABLE temp_file_option_values (
fileID INT,
optionID INT,
optionValue VARCHAR(255)
)');
DB::connection('mysql_hub')
->table('filebase1_file_option_value')
->orderBy('fileID')
->chunk(200, function ($options) {
foreach ($options as $option) {
DB::table('temp_file_option_values')->insert([
'fileID' => (int) $option->fileID,
'optionID' => (int) $option->optionID,
'optionValue' => $option->optionValue,
]);
}
});
}
/**
* Bring the file content from the Hub database to the local database temporary table.
*/
protected function bringFileContentLocal(): void
{
if (Schema::hasTable('temp_file_content')) {
DB::statement('DROP TABLE temp_file_content');
}
DB::statement('CREATE TEMPORARY TABLE temp_file_content (
fileID INT,
subject VARCHAR(255),
teaser VARCHAR(255),
message LONGTEXT
)');
DB::connection('mysql_hub')
->table('filebase1_file_content')
->orderBy('fileID')
->chunk(200, function ($contents) {
foreach ($contents as $content) {
DB::table('temp_file_content')->insert([
'fileID' => (int) $content->fileID,
'subject' => $content->subject,
'teaser' => $content->teaser,
'message' => $content->message,
]);
}
});
}
/**
* Bring the file version labels from the Hub database to the local database temporary table.
*/
protected function bringFileVersionLabelsLocal(): void
{
if (Schema::hasTable('temp_file_version_labels')) {
DB::statement('DROP TABLE temp_file_version_labels');
}
DB::statement('CREATE TEMPORARY TABLE temp_file_version_labels (
labelID INT,
objectID INT
)');
DB::connection('mysql_hub')
->table('wcf1_label_object')
->where('objectTypeID', 387)
->orderBy('labelID')
->chunk(200, function ($options) {
foreach ($options as $option) {
DB::table('temp_file_version_labels')->insert([
'labelID' => (int) $option->labelID,
'objectID' => (int) $option->objectID,
]);
}
});
}
/**
* Bring the file version content from the Hub database to the local database temporary table.
*/
protected function bringFileVersionContentLocal(): void
{
if (Schema::hasTable('temp_file_version_content')) {
DB::statement('DROP TABLE temp_file_version_content');
}
DB::statement('CREATE TEMPORARY TABLE temp_file_version_content (
versionID INT,
description TEXT
)');
DB::connection('mysql_hub')
->table('filebase1_file_version_content')
->orderBy('versionID')
->chunk(200, function ($options) {
foreach ($options as $option) {
DB::table('temp_file_version_content')->insert([
'versionID' => (int) $option->versionID,
'description' => $option->description,
]);
}
});
}
/**
* Import the users from the Hub database to the local database.
*/
protected function importUsers(): void
{
DB::connection('mysql_hub')
->table('wcf1_user')
->select('userID', 'username', 'email', 'password', 'registrationDate')
->chunkById(250, function (Collection $users) {
$insertData = [];
foreach ($users as $user) {
$insertData[] = [
'hub_id' => (int) $user->userID,
'name' => $user->username,
'email' => mb_convert_case($user->email, MB_CASE_LOWER, 'UTF-8'),
'password' => $this->cleanPasswordHash($user->password),
'created_at' => $this->cleanRegistrationDate($user->registrationDate),
'updated_at' => now('UTC')->toDateTimeString(),
];
}
if (! empty($insertData)) {
DB::table('users')->upsert(
$insertData,
['hub_id'],
['name', 'email', 'password', 'created_at', 'updated_at']
);
}
}, 'userID');
}
/**
* Clean the password hash from the Hub database.
*/
protected function cleanPasswordHash(string $password): string
{
// The hub passwords are hashed sometimes with a prefix of the hash type. We only want the hash.
// If it's not Bcrypt, they'll have to reset their password. Tough luck.
$clean = str_ireplace(['invalid:', 'bcrypt:', 'bcrypt::', 'cryptmd5:', 'cryptmd5::'], '', $password);
// If the password hash starts with $2, it's a valid Bcrypt hash. Otherwise, it's invalid.
return str_starts_with($clean, '$2') ? $clean : '';
}
/**
* Clean the registration date from the Hub database.
*/
protected function cleanRegistrationDate(string $registrationDate): string
{
$date = Carbon::createFromTimestamp($registrationDate);
// If the registration date is in the future, set it to now.
if ($date->isFuture()) {
$date = Carbon::now('UTC');
}
return $date->toDateTimeString();
}
/**
* Import the licenses from the Hub database to the local database.
*/
protected function importLicenses(): void
{
DB::connection('mysql_hub')
->table('filebase1_license')
->chunkById(100, function (Collection $licenses) {
$insertData = [];
foreach ($licenses as $license) {
$insertData[] = [
'hub_id' => (int) $license->licenseID,
'name' => $license->licenseName,
'link' => $license->licenseURL,
];
}
if (! empty($insertData)) {
DB::table('licenses')->upsert($insertData, ['hub_id'], ['name', 'link']);
}
}, 'licenseID');
}
/**
* Import the SPT versions from the Hub database to the local database.
*/
protected function importSptVersions(): void
{
DB::connection('mysql_hub')
->table('wcf1_label')
->where('groupID', 1)
->chunkById(100, function (Collection $versions) {
$insertData = [];
foreach ($versions as $version) {
$insertData[] = [
'hub_id' => (int) $version->labelID,
'version' => $version->label,
'color_class' => $this->translateColour($version->cssClassName),
];
}
if (! empty($insertData)) {
DB::table('spt_versions')->upsert($insertData, ['hub_id'], ['version', 'color_class']);
}
}, 'labelID');
}
/**
* Translate the colour class from the Hub database to the local database.
*/
protected function translateColour(string $colour = ''): string
{
return match ($colour) {
'green' => 'green',
'slightly-outdated' => 'lime',
'yellow' => 'yellow',
'red' => 'red',
default => 'gray',
};
}
/**
* Import the mods from the Hub database to the local database.
*/
protected function importMods(): void
{
// Initialize a cURL handler for downloading mod thumbnails.
$curl = curl_init();
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
DB::connection('mysql_hub')
->table('filebase1_file')
->chunkById(100, function (Collection $mods) use ($curl) {
foreach ($mods as $mod) {
$modContent = DB::table('temp_file_content')
->where('fileID', $mod->fileID)
->orderBy('fileID', 'desc')
->first();
$optionSourceCode = DB::table('temp_file_option_values')
->select('optionValue as source_code_link')
->where('fileID', $mod->fileID)
->whereIn('optionID', [5, 1])
->whereNot('optionValue', '')
->orderByDesc('optionID')
->first();
$optionContainsAi = DB::table('temp_file_option_values')
->select('optionValue as contains_ai')
->where('fileID', $mod->fileID)
->where('optionID', 7)
->whereNot('optionValue', '')
->first();
$optionContainsAds = DB::table('temp_file_option_values')
->select('optionValue as contains_ads')
->where('fileID', $mod->fileID)
->where('optionID', 3)
->whereNot('optionValue', '')
->first();
$versionLabel = DB::table('temp_file_version_labels')
->select('labelID')
->where('objectID', $mod->fileID)
->orderBy('labelID', 'desc')
->first();
// Skip the mod if it doesn't have a version label attached to it.
if (empty($versionLabel)) {
continue;
}
$insertData[] = [
'hub_id' => (int) $mod->fileID,
'user_id' => User::whereHubId($mod->userID)->value('id'),
'name' => $modContent?->subject ?? '',
'slug' => Str::slug($modContent?->subject ?? ''),
'teaser' => Str::limit($modContent?->teaser ?? ''),
'description' => $this->cleanHubContent($modContent?->message ?? ''),
'thumbnail' => $this->fetchModThumbnail($curl, $mod->fileID, $mod->iconHash, $mod->iconExtension),
'license_id' => License::whereHubId($mod->licenseID)->value('id'),
'source_code_link' => $optionSourceCode?->source_code_link ?? '',
'featured' => (bool) $mod?->isFeatured,
'contains_ai_content' => (bool) $optionContainsAi?->contains_ai,
'contains_ads' => (bool) $optionContainsAds?->contains_ads,
'disabled' => (bool) $mod?->isDisabled,
'created_at' => Carbon::parse($mod->time, 'UTC'),
'updated_at' => Carbon::parse($mod->lastChangeTime, 'UTC'),
];
}
if (! empty($insertData)) {
Mod::upsert($insertData, ['hub_id'], [
'user_id',
'name',
'slug',
'teaser',
'description',
'thumbnail',
'license_id',
'source_code_link',
'featured',
'contains_ai_content',
'disabled',
'created_at',
'updated_at',
]);
}
}, 'fileID');
// Close the cURL handler.
curl_close($curl);
}
/**
* Convert the mod description from WoltHub flavoured HTML to Markdown.
*/
protected function cleanHubContent(string $dirty): string
{
// Alright, hear me out... Shut up.
$converter = new HtmlConverter();
$clean = Purify::clean($dirty);
return $converter->convert($clean);
}
/**
* Fetch the mod thumbnail from the Hub and store it anew.
*/
protected function fetchModThumbnail(CurlHandle $curl, string $fileID, string $thumbnailHash, string $thumbnailExtension): string
{
// If any of the required fields are empty, return an empty string.
if (empty($fileID) || empty($thumbnailHash) || empty($thumbnailExtension)) {
return '';
}
// Build some paths/URLs using the mod data.
$hashShort = substr($thumbnailHash, 0, 2);
$fileName = $fileID.'.'.$thumbnailExtension;
$hubUrl = 'https://hub.sp-tarkov.com/files/images/file/'.$hashShort.'/'.$fileName;
$relativePath = 'mods/'.$fileName;
// Determine the disk to use based on the environment.
$disk = match (config('app.env')) {
'production' => 'r2', // Cloudflare R2 Storage
default => 'public', // Local
};
// Check to make sure the image doesn't already exist.
if (Storage::disk($disk)->exists($relativePath)) {
return $relativePath; // Already exists, return the path.
}
// Download the image using the cURL handler.
curl_setopt($curl, CURLOPT_URL, $hubUrl);
$image = curl_exec($curl);
if ($image === false) {
Log::error('There was an error attempting to download a mod thumbnail. cURL error: '.curl_error($curl));
return '';
}
// Store the image on the disk.
Storage::disk($disk)->put($relativePath, $image);
return $relativePath;
}
/**
* Import the mod versions from the Hub database to the local database.
*/
protected function importModVersions(): void
{
DB::connection('mysql_hub')
->table('filebase1_file_version')
->chunkById(500, function (Collection $versions) {
foreach ($versions as $version) {
$versionContent = DB::table('temp_file_version_content')
->select('description')
->where('versionID', $version->versionID)
->orderBy('versionID', 'desc')
->first();
$optionVirusTotal = DB::table('temp_file_option_values')
->select('optionValue as virus_total_link')
->where('fileID', $version->fileID)
->whereIn('optionID', [6, 2])
->whereNot('optionValue', '')
->orderByDesc('optionID')
->first();
$versionLabel = DB::table('temp_file_version_labels')
->select('labelID')
->where('objectID', $version->fileID)
->orderBy('labelID', 'desc')
->first();
$modId = Mod::whereHubId($version->fileID)->value('id');
// Skip the mod version if it doesn't have a mod or version label attached to it.
if (empty($versionLabel) || empty($modId)) {
continue;
}
$insertData[] = [
'hub_id' => (int) $version->versionID,
'mod_id' => $modId,
'version' => $version->versionNumber,
'description' => $this->cleanHubContent($versionContent->description ?? ''),
'link' => $version->downloadURL,
'spt_version_id' => SptVersion::whereHubId($versionLabel->labelID)->value('id'),
'virus_total_link' => $optionVirusTotal?->virus_total_link ?? '',
'downloads' => max((int) $version->downloads, 0), // At least 0.
'disabled' => (bool) $version->isDisabled,
'created_at' => Carbon::parse($version->uploadTime, 'UTC'),
'updated_at' => Carbon::parse($version->uploadTime, 'UTC'),
];
}
if (! empty($insertData)) {
ModVersion::upsert($insertData, ['hub_id'], [
'mod_id',
'version',
'description',
'link',
'spt_version_id',
'virus_total_link',
'downloads',
'created_at',
'updated_at',
]);
}
}, 'versionID');
}
/**
* The job failed to process.
*/
public function failed(Exception $exception): void
{
// Disconnect from the 'mysql_hub' database connection
DB::connection('mysql_hub')->disconnect();
}
}

2
config/database.php
View File

@ -190,7 +190,7 @@ return [
'username' => env('REDIS_USERNAME'),
'password' => env('REDIS_PASSWORD'),
'port' => env('REDIS_PORT', '6379'),
'database' => env('REDIS_CACHE_DB', '2'),
'database' => env('REDIS_QUEUE_DB', '2'),
],
],

32
config/horizon.php
View File

@ -182,8 +182,10 @@ return [
'defaults' => [
'supervisor-1' => [
'connection' => 'redis',
'queue' => ['short', 'long-running'],
'queue' => ['default'],
'balance' => 'auto',
'balanceMaxShift' => 1,
'balanceCooldown' => 3,
'autoScalingStrategy' => 'time',
'maxProcesses' => 1,
'maxTime' => 0,
@ -193,22 +195,36 @@ return [
'timeout' => 60,
'nice' => 0,
],
'supervisor-long' => [
'connection' => 'redis',
'queue' => ['long'],
'balance' => 'auto',
'balanceMaxShift' => 1,
'balanceCooldown' => 3,
'autoScalingStrategy' => 'time',
'maxProcesses' => 1,
'maxTime' => 0,
'maxJobs' => 0,
'memory' => 256,
'tries' => 1,
'timeout' => 900, // 15 Minutes
'nice' => 0,
],
],
'environments' => [
'production' => [
'supervisor-1' => [
'minProcesses' => 1,
'maxProcesses' => 10,
'balanceMaxShift' => 1,
'balanceCooldown' => 3,
'maxProcesses' => 12,
],
'supervisor-long' => [
'maxProcesses' => 4,
],
],
'local' => [
'supervisor-1' => [
'maxProcesses' => 3,
],
'supervisor-1' => [],
'supervisor-long' => [],
],
],
];

2
config/queue.php
View File

@ -66,7 +66,7 @@ return [
'redis' => [
'driver' => 'redis',
'connection' => env('REDIS_QUEUE_CONNECTION', 'queue'),
'queue' => env('REDIS_QUEUE', 'short'),
'queue' => env('REDIS_QUEUE', 'default'),
'retry_after' => (int) env('REDIS_QUEUE_RETRY_AFTER', 90),
'block_for' => null,
'after_commit' => false,

38
database/migrations/2024_06_10_002858_create_mod_indexes.php Normal file
View File

@ -0,0 +1,38 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('mods', function (Blueprint $table) {
$table->index(['deleted_at', 'disabled'], 'mods_show_index');
});
Schema::table('mod_versions', function (Blueprint $table) {
$table->index(['mod_id', 'deleted_at', 'disabled', 'version'], 'mod_versions_filtering_index');
});
Schema::table('spt_versions', function (Blueprint $table) {
$table->index(['version', 'deleted_at'], 'spt_versions_filtering_index');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('mods', function (Blueprint $table) {
$table->dropIndex('mods_show_index');
$table->dropIndex('mod_versions_filtering_index');
$table->dropIndex('spt_versions_filtering_index');
});
}
};

2
resources/views/components/mod-list.blade.php
View File

@ -10,7 +10,7 @@
<img src="https://placehold.co/450x450/EEE/31343C?font=source-sans-pro&text={{ $mod->name }}" alt="{{ $mod->name }}" class="block dark:hidden h-48 w-full object-cover md:h-full md:w-48 transform group-hover:scale-110 transition-all duration-200">
<img src="https://placehold.co/450x450/31343C/EEE?font=source-sans-pro&text={{ $mod->name }}" alt="{{ $mod->name }}" class="hidden dark:block h-48 w-full object-cover md:h-full md:w-48 transform group-hover:scale-110 transition-all duration-200">
@else
<img src="{{ Storage::url($mod->thumbnail) }}" alt="{{ $mod->name }}" class="h-48 w-full object-cover md:h-full md:w-48 transform group-hover:scale-110 transition-all duration-200">
<img src="{{ asset($mod->thumbnail) }}" alt="{{ $mod->name }}" class="h-48 w-full object-cover md:h-full md:w-48 transform group-hover:scale-110 transition-all duration-200">
@endif
</div>
<div class="flex flex-col w-full justify-between p-5">

2
resources/views/layouts/app.blade.php
View File

@ -11,6 +11,8 @@
<link href="//fonts.bunny.net" rel="preconnect">
<link href="//fonts.bunny.net/css?family=figtree:400,500,600&display=swap" rel="stylesheet">
<link href="{{ config('app.asset_url') }}" rel="dns-prefetch">
<script>
// Immediately set the theme to prevent a flash of the default theme when another is set.
// Must be located inline, in the head, and before any CSS is loaded.