diff --git a/app/Console/Commands/ImportHub.php b/app/Console/Commands/ImportHub.php index de1f524..7a95264 100644 --- a/app/Console/Commands/ImportHub.php +++ b/app/Console/Commands/ImportHub.php @@ -2,20 +2,8 @@ namespace App\Console\Commands; -use App\Models\License; -use App\Models\Mod; -use App\Models\ModVersion; -use App\Models\SptVersion; -use App\Models\User; -use Carbon\Carbon; +use App\Jobs\ImportHubData; use Illuminate\Console\Command; -use Illuminate\Support\Benchmark; -use Illuminate\Support\Collection; -use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\Storage; -use Illuminate\Support\Str; -use League\HTMLToMarkdown\HtmlConverter; -use Stevebauman\Purify\Facades\Purify; class ImportHub extends Command { @@ -23,469 +11,11 @@ class ImportHub extends Command protected $description = 'Connects to the Hub database and imports the data into the Laravel database.'; - /** - * Execute the console command. - */ public function handle(): void { - // This may take a minute or two... - set_time_limit(0); + // Add the ImportHubData job to the queue. + ImportHubData::dispatch()->onQueue('long'); - $this->newLine(); - - $totalTime = Benchmark::value(function () { - $loadDataTime = Benchmark::value(function () { - $this->loadData(); - }); - $this->info('Execution time: '.round($loadDataTime[1], 2).'ms'); - $this->newLine(); - - $importUsersTime = Benchmark::value(function () { - $this->importUsers(); - }); - $this->info('Execution time: '.round($importUsersTime[1], 2).'ms'); - $this->newLine(); - - $importLicensesTime = Benchmark::value(function () { - $this->importLicenses(); - }); - $this->info('Execution time: '.round($importLicensesTime[1], 2).'ms'); - $this->newLine(); - - $importSptVersionsTime = Benchmark::value(function () { - $this->importSptVersions(); - }); - $this->info('Execution time: '.round($importSptVersionsTime[1], 2).'ms'); - $this->newLine(); - - $importModsTime = Benchmark::value(function () { - $this->importMods(); - }); - $this->info('Execution time: '.round($importModsTime[1], 2).'ms'); - $this->newLine(); - - $importModVersionsTime = Benchmark::value(function () { - $this->importModVersions(); - }); - $this->info('Execution time: '.round($importModVersionsTime[1], 2).'ms'); - $this->newLine(); - }); - - // Disconnect from the Hub database, clearing temporary tables. - DB::connection('mysql_hub')->disconnect(); - - $this->newLine(); - $this->info('Data imported successfully'); - $this->info('Total execution time: '.round($totalTime[1], 2).'ms'); - - $this->newLine(); - $this->info('Refreshing Meilisearch indexes...'); - $this->call('scout:delete-all-indexes'); - $this->call('scout:sync-index-settings'); - $this->call('scout:import', ['model' => '\App\Models\Mod']); - - $this->newLine(); - $this->info('Done'); - } - - protected function loadData(): void - { - // We're just going to dump a few things in memory to escape the N+1 problem. - $this->output->write('Loading data into memory... '); - $this->bringFileOptionsLocal(); - $this->bringFileContentLocal(); - $this->bringFileVersionLabelsLocal(); - $this->bringFileVersionContentLocal(); - $this->info('Done.'); - } - - protected function importUsers(): void - { - $totalInserted = 0; - - foreach (DB::connection('mysql_hub')->table('wcf1_user')->orderBy('userID')->cursor() as $wolt) { - $registrationDate = Carbon::parse($wolt->registrationDate, 'UTC'); - if ($registrationDate->isFuture()) { - $registrationDate = now('UTC'); - } - $registrationDate->setTimezone('UTC'); - - $insertData = [ - 'hub_id' => $wolt->userID, - 'name' => $wolt->username, - 'email' => mb_convert_case($wolt->email, MB_CASE_LOWER, 'UTF-8'), - 'password' => $this->cleanPasswordHash($wolt->password), - 'created_at' => $registrationDate, - 'updated_at' => now('UTC')->toDateTimeString(), - ]; - - User::upsert($insertData, ['hub_id'], ['name', 'email', 'password', 'created_at', 'updated_at']); - $totalInserted++; - - // Log every 2500 users processed - if ($totalInserted % 2500 == 0) { - $this->line('Processed 2500 users. Total processed so far: '.$totalInserted); - } - } - - $this->info('Total users processed: '.$totalInserted); - } - - protected function cleanPasswordHash(string $password): string - { - // The hub passwords are hashed sometimes with a prefix of the hash type. We only want the hash. - // If it's not Bcrypt, they'll have to reset their password. Tough luck. - return str_replace(['Bcrypt:', 'cryptMD5:', 'cryptMD5::'], '', $password); - } - - protected function importLicenses(): void - { - $totalInserted = 0; - - DB::connection('mysql_hub') - ->table('filebase1_license') - ->chunkById(100, function (Collection $licenses) use (&$totalInserted) { - $insertData = []; - foreach ($licenses as $license) { - $insertData[] = [ - 'hub_id' => $license->licenseID, - 'name' => $license->licenseName, - 'link' => $license->licenseURL, - ]; - } - - if (! empty($insertData)) { - DB::table('licenses')->upsert($insertData, ['hub_id'], ['name', 'link']); - $totalInserted += count($insertData); - $this->line('Processed '.count($insertData).' licenses. Total processed so far: '.$totalInserted); - } - - unset($insertData); - unset($licenses); - }, 'licenseID'); - - $this->info('Total licenses processed: '.$totalInserted); - } - - protected function importSptVersions(): void - { - $totalInserted = 0; - - DB::connection('mysql_hub') - ->table('wcf1_label') - ->where('groupID', 1) - ->chunkById(100, function (Collection $versions) use (&$totalInserted) { - $insertData = []; - foreach ($versions as $version) { - $insertData[] = [ - 'hub_id' => $version->labelID, - 'version' => $version->label, - 'color_class' => $this->translateColour($version->cssClassName), - ]; - } - - if (! empty($insertData)) { - DB::table('spt_versions')->upsert($insertData, ['hub_id'], ['version', 'color_class']); - $totalInserted += count($insertData); - $this->line('Processed '.count($insertData).' SPT Versions. Total processed so far: '.$totalInserted); - } - - unset($insertData); - unset($versions); - }, 'labelID'); - - $this->info('Total licenses processed: '.$totalInserted); - } - - protected function translateColour(string $colour = ''): string - { - return match ($colour) { - 'green' => 'green', - 'slightly-outdated' => 'lime', - 'yellow' => 'yellow', - 'red' => 'red', - default => 'gray', - }; - } - - protected function importMods(): void - { - $command = $this; - $totalInserted = 0; - - $curl = curl_init(); - curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); - curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); - - DB::connection('mysql_hub') - ->table('filebase1_file') - ->chunkById(100, function (Collection $mods) use (&$command, &$curl, &$totalInserted) { - - foreach ($mods as $mod) { - - $modContent = DB::table('temp_file_content') - ->where('fileID', $mod->fileID) - ->orderBy('fileID', 'desc') - ->first(); - - $optionSourceCode = DB::table('temp_file_option_values') - ->select('optionValue as source_code_link') - ->where('fileID', $mod->fileID) - ->whereIn('optionID', [5, 1]) - ->whereNot('optionValue', '') - ->orderByDesc('optionID') - ->first(); - - $optionContainsAi = DB::table('temp_file_option_values') - ->select('optionValue as contains_ai') - ->where('fileID', $mod->fileID) - ->where('optionID', 7) - ->whereNot('optionValue', '') - ->first(); - - $optionContainsAds = DB::table('temp_file_option_values') - ->select('optionValue as contains_ads') - ->where('fileID', $mod->fileID) - ->where('optionID', 3) - ->whereNot('optionValue', '') - ->first(); - - $versionLabel = DB::table('temp_file_version_labels') - ->select('labelID') - ->where('objectID', $mod->fileID) - ->orderBy('labelID', 'desc') - ->first(); - - if (empty($versionLabel)) { - continue; - } - - $insertData[] = [ - 'hub_id' => (int) $mod->fileID, - 'user_id' => User::whereHubId($mod->userID)->value('id'), - 'name' => $modContent?->subject ?? '', - 'slug' => Str::slug($modContent?->subject) ?? '', - 'teaser' => Str::limit($modContent?->teaser) ?? '', - 'description' => $this->convertModDescription($modContent?->message ?? ''), - 'thumbnail' => $this->fetchModThumbnail($command, $curl, $mod->fileID, $mod->iconHash, $mod->iconExtension), - 'license_id' => License::whereHubId($mod->licenseID)->value('id'), - 'source_code_link' => $optionSourceCode?->source_code_link ?? '', - 'featured' => (bool) $mod->isFeatured, - 'contains_ai_content' => (bool) $optionContainsAi?->contains_ai ?? false, - 'contains_ads' => (bool) $optionContainsAds?->contains_ads ?? false, - 'disabled' => (bool) $mod->isDisabled, - 'created_at' => Carbon::parse($mod->time, 'UTC'), - 'updated_at' => Carbon::parse($mod->lastChangeTime, 'UTC'), - ]; - } - - if (! empty($insertData)) { - Mod::upsert($insertData, ['hub_id'], ['user_id', 'name', 'slug', 'teaser', 'description', 'thumbnail', 'license_id', 'source_code_link', 'featured', 'contains_ai_content', 'disabled', 'created_at', 'updated_at']); - $totalInserted += count($insertData); - $command->line('Processed '.count($insertData).' mods. Total processed so far: '.$totalInserted); - } - - unset($insertData); - unset($mods); - }, 'fileID'); - - curl_close($curl); - - $this->info('Total mods processed: '.$totalInserted); - } - - protected function bringFileOptionsLocal(): void - { - DB::statement('CREATE TEMPORARY TABLE temp_file_option_values ( - fileID INT, - optionID INT, - optionValue VARCHAR(255) - )'); - - DB::connection('mysql_hub') - ->table('filebase1_file_option_value') - ->orderBy('fileID') - ->chunk(200, function ($options) { - foreach ($options as $option) { - DB::table('temp_file_option_values')->insert([ - 'fileID' => $option->fileID, - 'optionID' => $option->optionID, - 'optionValue' => $option->optionValue, - ]); - } - }); - } - - protected function bringFileContentLocal(): void - { - DB::statement('CREATE TEMPORARY TABLE temp_file_content ( - fileID INT, - subject VARCHAR(255), - teaser VARCHAR(255), - message LONGTEXT - )'); - - DB::connection('mysql_hub') - ->table('filebase1_file_content') - ->orderBy('fileID') - ->chunk(200, function ($contents) { - foreach ($contents as $content) { - DB::table('temp_file_content')->insert([ - 'fileID' => $content->fileID, - 'subject' => $content->subject, - 'teaser' => $content->teaser, - 'message' => $content->message, - ]); - } - }); - } - - protected function fetchModThumbnail($command, $curl, string $fileID, string $thumbnailHash, string $thumbnailExtension): string - { - if (empty($fileID) || empty($thumbnailHash) || empty($thumbnailExtension)) { - return ''; - } - - // Only the first two characters of the icon hash. - $hashShort = substr($thumbnailHash, 0, 2); - - $hubUrl = "https://hub.sp-tarkov.com/files/images/file/$hashShort/$fileID.$thumbnailExtension"; - $relativePath = "mods/$thumbnailHash.$thumbnailExtension"; - - $disk = match (config('app.env')) { - 'production' => 'r2', - default => 'local', - }; - - // Check to make sure the image doesn't already exist. - if (Storage::disk($disk)->exists($relativePath)) { - return $relativePath; - } - - $command->output->write("Downloading mod thumbnail: $hubUrl... "); - curl_setopt($curl, CURLOPT_URL, $hubUrl); - $image = curl_exec($curl); - if ($image === false) { - $command->error('Error: '.curl_error($curl)); - } else { - Storage::disk($disk)->put($relativePath, $image); - $command->info('Done.'); - } - - return $relativePath; - } - - protected function importModVersions(): void - { - $command = $this; - $totalInserted = 0; - - DB::connection('mysql_hub') - ->table('filebase1_file_version') - ->chunkById(500, function (Collection $versions) use (&$command, &$totalInserted) { - - foreach ($versions as $version) { - - $versionContent = DB::table('temp_file_version_content') - ->select('description') - ->where('versionID', $version->versionID) - ->orderBy('versionID', 'desc') - ->first(); - - $optionVirusTotal = DB::table('temp_file_option_values') - ->select('optionValue as virus_total_link') - ->where('fileID', $version->fileID) - ->whereIn('optionID', [6, 2]) - ->whereNot('optionValue', '') - ->orderByDesc('optionID') - ->first(); - - $versionLabel = DB::table('temp_file_version_labels') - ->select('labelID') - ->where('objectID', $version->fileID) - ->orderBy('labelID', 'desc') - ->first(); - - $modId = Mod::whereHubId($version->fileID)->value('id'); - - if (empty($versionLabel) || empty($modId)) { - continue; - } - - $insertData[] = [ - 'hub_id' => $version->versionID, - 'mod_id' => $modId, - 'version' => $version->versionNumber, - 'description' => $this->convertModDescription($versionContent->description ?? ''), - 'link' => $version->downloadURL, - 'spt_version_id' => SptVersion::whereHubId($versionLabel->labelID)->value('id'), - 'virus_total_link' => $optionVirusTotal?->virus_total_link ?? '', - 'downloads' => max((int) $version->downloads, 0), // Ensure the value is at least 0 - 'disabled' => (bool) $version->isDisabled, - 'created_at' => Carbon::parse($version->uploadTime, 'UTC'), - 'updated_at' => Carbon::parse($version->uploadTime, 'UTC'), - ]; - } - - if (! empty($insertData)) { - ModVersion::upsert($insertData, ['hub_id'], ['mod_id', 'version', 'description', 'link', 'spt_version_id', 'virus_total_link', 'downloads', 'created_at', 'updated_at']); - $totalInserted += count($insertData); - $command->line('Processed '.count($insertData).' mod versions. Total processed so far: '.$totalInserted); - } - - unset($insertData); - unset($version); - }, 'versionID'); - - $this->info('Total mod versions processed: '.$totalInserted); - } - - protected function bringFileVersionLabelsLocal(): void - { - DB::statement('CREATE TEMPORARY TABLE temp_file_version_labels ( - labelID INT, - objectID INT - )'); - - DB::connection('mysql_hub') - ->table('wcf1_label_object') - ->where('objectTypeID', 387) - ->orderBy('labelID') - ->chunk(200, function ($options) { - foreach ($options as $option) { - DB::table('temp_file_version_labels')->insert([ - 'labelID' => $option->labelID, - 'objectID' => $option->objectID, - ]); - } - }); - } - - protected function bringFileVersionContentLocal(): void - { - DB::statement('CREATE TEMPORARY TABLE temp_file_version_content ( - versionID INT, - description TEXT - )'); - - DB::connection('mysql_hub') - ->table('filebase1_file_version_content') - ->orderBy('versionID') - ->chunk(200, function ($options) { - foreach ($options as $option) { - DB::table('temp_file_version_content')->insert([ - 'versionID' => $option->versionID, - 'description' => $option->description, - ]); - } - }); - } - - protected function convertModDescription(string $description): string - { - // Alright, hear me out... Shut up. - $converter = new HtmlConverter(); - - return $converter->convert(Purify::clean($description)); + $this->info('The import job has been added to the queue.'); } } diff --git a/app/Console/Commands/UploadAssets.php b/app/Console/Commands/UploadAssets.php index 02a33c4..86e1906 100644 --- a/app/Console/Commands/UploadAssets.php +++ b/app/Console/Commands/UploadAssets.php @@ -8,24 +8,15 @@ use Illuminate\Support\Facades\Storage; class UploadAssets extends Command { - /** - * The name and signature of the console command. - * - * @var string - */ protected $signature = 'app:upload-assets'; - /** - * The console command description. - * - * @var string - */ protected $description = 'Uploads the Vite build assets to Cloudflare R2'; /** - * Execute the console command. + * This command uploads the Vite build assets to Cloudflare R2. Typically, this will be run after the assets have + * been built and the application is ready to deploy from within the production environment build process. */ - public function handle() + public function handle(): void { $this->info('Publishing assets...'); diff --git a/app/Jobs/ImportHubData.php b/app/Jobs/ImportHubData.php new file mode 100644 index 0000000..857ea44 --- /dev/null +++ b/app/Jobs/ImportHubData.php @@ -0,0 +1,518 @@ +bringFileOptionsLocal(); + $this->bringFileContentLocal(); + $this->bringFileVersionLabelsLocal(); + $this->bringFileVersionContentLocal(); + + // Begin to import the data into the permanent local database tables. + $this->importUsers(); + $this->importLicenses(); + $this->importSptVersions(); + $this->importMods(); + $this->importModVersions(); + + // Ensure that we've disconnected from the Hub database, clearing temporary tables. + DB::connection('mysql_hub')->disconnect(); + + // Reindex the Meilisearch index. + Artisan::call('scout:delete-all-indexes'); + Artisan::call('scout:sync-index-settings'); + Artisan::call('scout:import', ['model' => '\App\Models\Mod']); + } + + /** + * Bring the file options from the Hub database to the local database temporary table. + */ + protected function bringFileOptionsLocal(): void + { + if (Schema::hasTable('temp_file_option_values')) { + DB::statement('DROP TABLE temp_file_option_values'); + } + DB::statement('CREATE TEMPORARY TABLE temp_file_option_values ( + fileID INT, + optionID INT, + optionValue VARCHAR(255) + )'); + + DB::connection('mysql_hub') + ->table('filebase1_file_option_value') + ->orderBy('fileID') + ->chunk(200, function ($options) { + foreach ($options as $option) { + DB::table('temp_file_option_values')->insert([ + 'fileID' => (int) $option->fileID, + 'optionID' => (int) $option->optionID, + 'optionValue' => $option->optionValue, + ]); + } + }); + } + + /** + * Bring the file content from the Hub database to the local database temporary table. + */ + protected function bringFileContentLocal(): void + { + if (Schema::hasTable('temp_file_content')) { + DB::statement('DROP TABLE temp_file_content'); + } + DB::statement('CREATE TEMPORARY TABLE temp_file_content ( + fileID INT, + subject VARCHAR(255), + teaser VARCHAR(255), + message LONGTEXT + )'); + + DB::connection('mysql_hub') + ->table('filebase1_file_content') + ->orderBy('fileID') + ->chunk(200, function ($contents) { + foreach ($contents as $content) { + DB::table('temp_file_content')->insert([ + 'fileID' => (int) $content->fileID, + 'subject' => $content->subject, + 'teaser' => $content->teaser, + 'message' => $content->message, + ]); + } + }); + } + + /** + * Bring the file version labels from the Hub database to the local database temporary table. + */ + protected function bringFileVersionLabelsLocal(): void + { + if (Schema::hasTable('temp_file_version_labels')) { + DB::statement('DROP TABLE temp_file_version_labels'); + } + DB::statement('CREATE TEMPORARY TABLE temp_file_version_labels ( + labelID INT, + objectID INT + )'); + + DB::connection('mysql_hub') + ->table('wcf1_label_object') + ->where('objectTypeID', 387) + ->orderBy('labelID') + ->chunk(200, function ($options) { + foreach ($options as $option) { + DB::table('temp_file_version_labels')->insert([ + 'labelID' => (int) $option->labelID, + 'objectID' => (int) $option->objectID, + ]); + } + }); + } + + /** + * Bring the file version content from the Hub database to the local database temporary table. + */ + protected function bringFileVersionContentLocal(): void + { + if (Schema::hasTable('temp_file_version_content')) { + DB::statement('DROP TABLE temp_file_version_content'); + } + DB::statement('CREATE TEMPORARY TABLE temp_file_version_content ( + versionID INT, + description TEXT + )'); + + DB::connection('mysql_hub') + ->table('filebase1_file_version_content') + ->orderBy('versionID') + ->chunk(200, function ($options) { + foreach ($options as $option) { + DB::table('temp_file_version_content')->insert([ + 'versionID' => (int) $option->versionID, + 'description' => $option->description, + ]); + } + }); + } + + /** + * Import the users from the Hub database to the local database. + */ + protected function importUsers(): void + { + DB::connection('mysql_hub') + ->table('wcf1_user') + ->select('userID', 'username', 'email', 'password', 'registrationDate') + ->chunkById(250, function (Collection $users) { + + $insertData = []; + foreach ($users as $user) { + $insertData[] = [ + 'hub_id' => (int) $user->userID, + 'name' => $user->username, + 'email' => mb_convert_case($user->email, MB_CASE_LOWER, 'UTF-8'), + 'password' => $this->cleanPasswordHash($user->password), + 'created_at' => $this->cleanRegistrationDate($user->registrationDate), + 'updated_at' => now('UTC')->toDateTimeString(), + ]; + } + + if (! empty($insertData)) { + DB::table('users')->upsert( + $insertData, + ['hub_id'], + ['name', 'email', 'password', 'created_at', 'updated_at'] + ); + } + }, 'userID'); + } + + /** + * Clean the password hash from the Hub database. + */ + protected function cleanPasswordHash(string $password): string + { + // The hub passwords are hashed sometimes with a prefix of the hash type. We only want the hash. + // If it's not Bcrypt, they'll have to reset their password. Tough luck. + $clean = str_ireplace(['invalid:', 'bcrypt:', 'bcrypt::', 'cryptmd5:', 'cryptmd5::'], '', $password); + + // If the password hash starts with $2, it's a valid Bcrypt hash. Otherwise, it's invalid. + return str_starts_with($clean, '$2') ? $clean : ''; + } + + /** + * Clean the registration date from the Hub database. + */ + protected function cleanRegistrationDate(string $registrationDate): string + { + $date = Carbon::createFromTimestamp($registrationDate); + + // If the registration date is in the future, set it to now. + if ($date->isFuture()) { + $date = Carbon::now('UTC'); + } + + return $date->toDateTimeString(); + } + + /** + * Import the licenses from the Hub database to the local database. + */ + protected function importLicenses(): void + { + DB::connection('mysql_hub') + ->table('filebase1_license') + ->chunkById(100, function (Collection $licenses) { + + $insertData = []; + foreach ($licenses as $license) { + $insertData[] = [ + 'hub_id' => (int) $license->licenseID, + 'name' => $license->licenseName, + 'link' => $license->licenseURL, + ]; + } + + if (! empty($insertData)) { + DB::table('licenses')->upsert($insertData, ['hub_id'], ['name', 'link']); + } + }, 'licenseID'); + } + + /** + * Import the SPT versions from the Hub database to the local database. + */ + protected function importSptVersions(): void + { + DB::connection('mysql_hub') + ->table('wcf1_label') + ->where('groupID', 1) + ->chunkById(100, function (Collection $versions) { + $insertData = []; + foreach ($versions as $version) { + $insertData[] = [ + 'hub_id' => (int) $version->labelID, + 'version' => $version->label, + 'color_class' => $this->translateColour($version->cssClassName), + ]; + } + + if (! empty($insertData)) { + DB::table('spt_versions')->upsert($insertData, ['hub_id'], ['version', 'color_class']); + } + }, 'labelID'); + } + + /** + * Translate the colour class from the Hub database to the local database. + */ + protected function translateColour(string $colour = ''): string + { + return match ($colour) { + 'green' => 'green', + 'slightly-outdated' => 'lime', + 'yellow' => 'yellow', + 'red' => 'red', + default => 'gray', + }; + } + + /** + * Import the mods from the Hub database to the local database. + */ + protected function importMods(): void + { + // Initialize a cURL handler for downloading mod thumbnails. + $curl = curl_init(); + curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); + curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); + + DB::connection('mysql_hub') + ->table('filebase1_file') + ->chunkById(100, function (Collection $mods) use ($curl) { + + foreach ($mods as $mod) { + $modContent = DB::table('temp_file_content') + ->where('fileID', $mod->fileID) + ->orderBy('fileID', 'desc') + ->first(); + + $optionSourceCode = DB::table('temp_file_option_values') + ->select('optionValue as source_code_link') + ->where('fileID', $mod->fileID) + ->whereIn('optionID', [5, 1]) + ->whereNot('optionValue', '') + ->orderByDesc('optionID') + ->first(); + + $optionContainsAi = DB::table('temp_file_option_values') + ->select('optionValue as contains_ai') + ->where('fileID', $mod->fileID) + ->where('optionID', 7) + ->whereNot('optionValue', '') + ->first(); + + $optionContainsAds = DB::table('temp_file_option_values') + ->select('optionValue as contains_ads') + ->where('fileID', $mod->fileID) + ->where('optionID', 3) + ->whereNot('optionValue', '') + ->first(); + + $versionLabel = DB::table('temp_file_version_labels') + ->select('labelID') + ->where('objectID', $mod->fileID) + ->orderBy('labelID', 'desc') + ->first(); + + // Skip the mod if it doesn't have a version label attached to it. + if (empty($versionLabel)) { + continue; + } + + $insertData[] = [ + 'hub_id' => (int) $mod->fileID, + 'user_id' => User::whereHubId($mod->userID)->value('id'), + 'name' => $modContent?->subject ?? '', + 'slug' => Str::slug($modContent?->subject ?? ''), + 'teaser' => Str::limit($modContent?->teaser ?? ''), + 'description' => $this->cleanHubContent($modContent?->message ?? ''), + 'thumbnail' => $this->fetchModThumbnail($curl, $mod->fileID, $mod->iconHash, $mod->iconExtension), + 'license_id' => License::whereHubId($mod->licenseID)->value('id'), + 'source_code_link' => $optionSourceCode?->source_code_link ?? '', + 'featured' => (bool) $mod?->isFeatured, + 'contains_ai_content' => (bool) $optionContainsAi?->contains_ai, + 'contains_ads' => (bool) $optionContainsAds?->contains_ads, + 'disabled' => (bool) $mod?->isDisabled, + 'created_at' => Carbon::parse($mod->time, 'UTC'), + 'updated_at' => Carbon::parse($mod->lastChangeTime, 'UTC'), + ]; + } + + if (! empty($insertData)) { + Mod::upsert($insertData, ['hub_id'], [ + 'user_id', + 'name', + 'slug', + 'teaser', + 'description', + 'thumbnail', + 'license_id', + 'source_code_link', + 'featured', + 'contains_ai_content', + 'disabled', + 'created_at', + 'updated_at', + ]); + } + }, 'fileID'); + + // Close the cURL handler. + curl_close($curl); + } + + /** + * Convert the mod description from WoltHub flavoured HTML to Markdown. + */ + protected function cleanHubContent(string $dirty): string + { + // Alright, hear me out... Shut up. + + $converter = new HtmlConverter(); + $clean = Purify::clean($dirty); + + return $converter->convert($clean); + } + + /** + * Fetch the mod thumbnail from the Hub and store it anew. + */ + protected function fetchModThumbnail(CurlHandle $curl, string $fileID, string $thumbnailHash, string $thumbnailExtension): string + { + // If any of the required fields are empty, return an empty string. + if (empty($fileID) || empty($thumbnailHash) || empty($thumbnailExtension)) { + return ''; + } + + // Build some paths/URLs using the mod data. + $hashShort = substr($thumbnailHash, 0, 2); + $fileName = $fileID.'.'.$thumbnailExtension; + $hubUrl = 'https://hub.sp-tarkov.com/files/images/file/'.$hashShort.'/'.$fileName; + $relativePath = 'mods/'.$fileName; + + // Determine the disk to use based on the environment. + $disk = match (config('app.env')) { + 'production' => 'r2', // Cloudflare R2 Storage + default => 'public', // Local + }; + + // Check to make sure the image doesn't already exist. + if (Storage::disk($disk)->exists($relativePath)) { + return $relativePath; // Already exists, return the path. + } + + // Download the image using the cURL handler. + curl_setopt($curl, CURLOPT_URL, $hubUrl); + $image = curl_exec($curl); + + if ($image === false) { + Log::error('There was an error attempting to download a mod thumbnail. cURL error: '.curl_error($curl)); + + return ''; + } + + // Store the image on the disk. + Storage::disk($disk)->put($relativePath, $image); + + return $relativePath; + } + + /** + * Import the mod versions from the Hub database to the local database. + */ + protected function importModVersions(): void + { + DB::connection('mysql_hub') + ->table('filebase1_file_version') + ->chunkById(500, function (Collection $versions) { + + foreach ($versions as $version) { + $versionContent = DB::table('temp_file_version_content') + ->select('description') + ->where('versionID', $version->versionID) + ->orderBy('versionID', 'desc') + ->first(); + + $optionVirusTotal = DB::table('temp_file_option_values') + ->select('optionValue as virus_total_link') + ->where('fileID', $version->fileID) + ->whereIn('optionID', [6, 2]) + ->whereNot('optionValue', '') + ->orderByDesc('optionID') + ->first(); + + $versionLabel = DB::table('temp_file_version_labels') + ->select('labelID') + ->where('objectID', $version->fileID) + ->orderBy('labelID', 'desc') + ->first(); + + $modId = Mod::whereHubId($version->fileID)->value('id'); + + // Skip the mod version if it doesn't have a mod or version label attached to it. + if (empty($versionLabel) || empty($modId)) { + continue; + } + + $insertData[] = [ + 'hub_id' => (int) $version->versionID, + 'mod_id' => $modId, + 'version' => $version->versionNumber, + 'description' => $this->cleanHubContent($versionContent->description ?? ''), + 'link' => $version->downloadURL, + 'spt_version_id' => SptVersion::whereHubId($versionLabel->labelID)->value('id'), + 'virus_total_link' => $optionVirusTotal?->virus_total_link ?? '', + 'downloads' => max((int) $version->downloads, 0), // At least 0. + 'disabled' => (bool) $version->isDisabled, + 'created_at' => Carbon::parse($version->uploadTime, 'UTC'), + 'updated_at' => Carbon::parse($version->uploadTime, 'UTC'), + ]; + } + + if (! empty($insertData)) { + ModVersion::upsert($insertData, ['hub_id'], [ + 'mod_id', + 'version', + 'description', + 'link', + 'spt_version_id', + 'virus_total_link', + 'downloads', + 'created_at', + 'updated_at', + ]); + } + }, 'versionID'); + } + + /** + * The job failed to process. + */ + public function failed(Exception $exception): void + { + // Disconnect from the 'mysql_hub' database connection + DB::connection('mysql_hub')->disconnect(); + } +} diff --git a/config/database.php b/config/database.php index c457ab1..d548da4 100644 --- a/config/database.php +++ b/config/database.php @@ -190,7 +190,7 @@ return [ 'username' => env('REDIS_USERNAME'), 'password' => env('REDIS_PASSWORD'), 'port' => env('REDIS_PORT', '6379'), - 'database' => env('REDIS_CACHE_DB', '2'), + 'database' => env('REDIS_QUEUE_DB', '2'), ], ], diff --git a/config/horizon.php b/config/horizon.php index 40d0502..361ecde 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -182,8 +182,10 @@ return [ 'defaults' => [ 'supervisor-1' => [ 'connection' => 'redis', - 'queue' => ['short', 'long-running'], + 'queue' => ['default'], 'balance' => 'auto', + 'balanceMaxShift' => 1, + 'balanceCooldown' => 3, 'autoScalingStrategy' => 'time', 'maxProcesses' => 1, 'maxTime' => 0, @@ -193,22 +195,36 @@ return [ 'timeout' => 60, 'nice' => 0, ], + 'supervisor-long' => [ + 'connection' => 'redis', + 'queue' => ['long'], + 'balance' => 'auto', + 'balanceMaxShift' => 1, + 'balanceCooldown' => 3, + 'autoScalingStrategy' => 'time', + 'maxProcesses' => 1, + 'maxTime' => 0, + 'maxJobs' => 0, + 'memory' => 256, + 'tries' => 1, + 'timeout' => 900, // 15 Minutes + 'nice' => 0, + ], ], 'environments' => [ 'production' => [ 'supervisor-1' => [ - 'minProcesses' => 1, - 'maxProcesses' => 10, - 'balanceMaxShift' => 1, - 'balanceCooldown' => 3, + 'maxProcesses' => 12, + ], + 'supervisor-long' => [ + 'maxProcesses' => 4, ], ], 'local' => [ - 'supervisor-1' => [ - 'maxProcesses' => 3, - ], + 'supervisor-1' => [], + 'supervisor-long' => [], ], ], ]; diff --git a/config/queue.php b/config/queue.php index 36079e4..11c0504 100644 --- a/config/queue.php +++ b/config/queue.php @@ -66,7 +66,7 @@ return [ 'redis' => [ 'driver' => 'redis', 'connection' => env('REDIS_QUEUE_CONNECTION', 'queue'), - 'queue' => env('REDIS_QUEUE', 'short'), + 'queue' => env('REDIS_QUEUE', 'default'), 'retry_after' => (int) env('REDIS_QUEUE_RETRY_AFTER', 90), 'block_for' => null, 'after_commit' => false, diff --git a/database/migrations/2024_06_10_002858_create_mod_indexes.php b/database/migrations/2024_06_10_002858_create_mod_indexes.php new file mode 100644 index 0000000..f675012 --- /dev/null +++ b/database/migrations/2024_06_10_002858_create_mod_indexes.php @@ -0,0 +1,38 @@ +index(['deleted_at', 'disabled'], 'mods_show_index'); + }); + + Schema::table('mod_versions', function (Blueprint $table) { + $table->index(['mod_id', 'deleted_at', 'disabled', 'version'], 'mod_versions_filtering_index'); + }); + + Schema::table('spt_versions', function (Blueprint $table) { + $table->index(['version', 'deleted_at'], 'spt_versions_filtering_index'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('mods', function (Blueprint $table) { + $table->dropIndex('mods_show_index'); + $table->dropIndex('mod_versions_filtering_index'); + $table->dropIndex('spt_versions_filtering_index'); + }); + } +}; diff --git a/resources/views/components/mod-list.blade.php b/resources/views/components/mod-list.blade.php index 20fea7e..ee1b199 100644 --- a/resources/views/components/mod-list.blade.php +++ b/resources/views/components/mod-list.blade.php @@ -10,7 +10,7 @@ {{ $mod->name }} @else - {{ $mod->name }} + {{ $mod->name }} @endif
diff --git a/resources/views/layouts/app.blade.php b/resources/views/layouts/app.blade.php index 4b984d3..11e17a4 100644 --- a/resources/views/layouts/app.blade.php +++ b/resources/views/layouts/app.blade.php @@ -11,6 +11,8 @@ + +