newLine(); $totalTime = Benchmark::value(function () { $loadDataTime = Benchmark::value(function () { $this->loadData(); }); $this->info('Execution time: '.round($loadDataTime[1], 2).'ms'); $this->newLine(); $importUsersTime = Benchmark::value(function () { $this->importUsers(); }); $this->info('Execution time: '.round($importUsersTime[1], 2).'ms'); $this->newLine(); $importLicensesTime = Benchmark::value(function () { $this->importLicenses(); }); $this->info('Execution time: '.round($importLicensesTime[1], 2).'ms'); $this->newLine(); $importSptVersionsTime = Benchmark::value(function () { $this->importSptVersions(); }); $this->info('Execution time: '.round($importSptVersionsTime[1], 2).'ms'); $this->newLine(); $importModsTime = Benchmark::value(function () { $this->importMods(); }); $this->info('Execution time: '.round($importModsTime[1], 2).'ms'); $this->newLine(); $importModVersionsTime = Benchmark::value(function () { $this->importModVersions(); }); $this->info('Execution time: '.round($importModVersionsTime[1], 2).'ms'); $this->newLine(); }); // Disconnect from the Hub database, clearing temporary tables. DB::connection('mysql_hub')->disconnect(); $this->newLine(); $this->info('Data imported successfully'); $this->info('Total execution time: '.round($totalTime[1], 2).'ms'); $this->newLine(); $this->info('Refreshing Meilisearch indexes...'); $this->call('scout:delete-all-indexes'); $this->call('scout:sync-index-settings'); $this->call('scout:import', ['model' => '\App\Models\Mod']); $this->newLine(); $this->info('Done'); } protected function loadData(): void { // We're just going to dump a few things in memory to escape the N+1 problem. $this->output->write('Loading data into memory... '); $this->bringFileOptionsLocal(); $this->bringFileContentLocal(); $this->bringFileVersionLabelsLocal(); $this->bringFileVersionContentLocal(); $this->info('Done.'); } protected function importUsers(): void { $totalInserted = 0; foreach (DB::connection('mysql_hub')->table('wcf1_user')->orderBy('userID')->cursor() as $wolt) { $registrationDate = Carbon::parse($wolt->registrationDate, 'UTC'); if ($registrationDate->isFuture()) { $registrationDate = now('UTC'); } $registrationDate->setTimezone('UTC'); $insertData = [ 'hub_id' => $wolt->userID, 'name' => $wolt->username, 'email' => mb_convert_case($wolt->email, MB_CASE_LOWER, 'UTF-8'), 'password' => $this->cleanPasswordHash($wolt->password), 'created_at' => $registrationDate, 'updated_at' => now('UTC')->toDateTimeString(), ]; User::upsert($insertData, ['hub_id'], ['name', 'email', 'password', 'created_at', 'updated_at']); $totalInserted++; // Log every 2500 users processed if ($totalInserted % 2500 == 0) { $this->line('Processed 2500 users. Total processed so far: '.$totalInserted); } } $this->info('Total users processed: '.$totalInserted); } protected function cleanPasswordHash(string $password): string { // The hub passwords are hashed sometimes with a prefix of the hash type. We only want the hash. // If it's not Bcrypt, they'll have to reset their password. Tough luck. return str_replace(['Bcrypt:', 'cryptMD5:', 'cryptMD5::'], '', $password); } protected function importLicenses(): void { $totalInserted = 0; DB::connection('mysql_hub') ->table('filebase1_license') ->chunkById(100, function (Collection $licenses) use (&$totalInserted) { $insertData = []; foreach ($licenses as $license) { $insertData[] = [ 'hub_id' => $license->licenseID, 'name' => $license->licenseName, 'link' => $license->licenseURL, ]; } if (! empty($insertData)) { DB::table('licenses')->upsert($insertData, ['hub_id'], ['name', 'link']); $totalInserted += count($insertData); $this->line('Processed '.count($insertData).' licenses. Total processed so far: '.$totalInserted); } unset($insertData); unset($licenses); }, 'licenseID'); $this->info('Total licenses processed: '.$totalInserted); } protected function importSptVersions(): void { $totalInserted = 0; DB::connection('mysql_hub') ->table('wcf1_label') ->where('groupID', 1) ->chunkById(100, function (Collection $versions) use (&$totalInserted) { $insertData = []; foreach ($versions as $version) { $insertData[] = [ 'hub_id' => $version->labelID, 'version' => $version->label, 'color_class' => $this->translateColour($version->cssClassName), ]; } if (! empty($insertData)) { DB::table('spt_versions')->upsert($insertData, ['hub_id'], ['version', 'color_class']); $totalInserted += count($insertData); $this->line('Processed '.count($insertData).' SPT Versions. Total processed so far: '.$totalInserted); } unset($insertData); unset($versions); }, 'labelID'); $this->info('Total licenses processed: '.$totalInserted); } protected function translateColour(string $colour = ''): string { return match ($colour) { 'green' => 'green', 'slightly-outdated' => 'lime', 'yellow' => 'yellow', 'red' => 'red', default => 'gray', }; } protected function importMods(): void { $command = $this; $totalInserted = 0; $curl = curl_init(); curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); DB::connection('mysql_hub') ->table('filebase1_file') ->chunkById(100, function (Collection $mods) use (&$command, &$curl, &$totalInserted) { foreach ($mods as $mod) { $modContent = DB::table('temp_file_content') ->where('fileID', $mod->fileID) ->orderBy('fileID', 'desc') ->first(); $optionSourceCode = DB::table('temp_file_option_values') ->select('optionValue as source_code_link') ->where('fileID', $mod->fileID) ->whereIn('optionID', [5, 1]) ->whereNot('optionValue', '') ->orderByDesc('optionID') ->first(); $optionContainsAi = DB::table('temp_file_option_values') ->select('optionValue as contains_ai') ->where('fileID', $mod->fileID) ->where('optionID', 7) ->whereNot('optionValue', '') ->first(); $optionContainsAds = DB::table('temp_file_option_values') ->select('optionValue as contains_ads') ->where('fileID', $mod->fileID) ->where('optionID', 3) ->whereNot('optionValue', '') ->first(); $versionLabel = DB::table('temp_file_version_labels') ->select('labelID') ->where('objectID', $mod->fileID) ->orderBy('labelID', 'desc') ->first(); if (empty($versionLabel)) { continue; } $insertData[] = [ 'hub_id' => (int) $mod->fileID, 'user_id' => User::whereHubId($mod->userID)->value('id'), 'name' => $modContent?->subject ?? '', 'slug' => Str::slug($modContent?->subject) ?? '', 'teaser' => Str::limit($modContent?->teaser) ?? '', 'description' => $this->convertModDescription($modContent?->message ?? ''), 'thumbnail' => $this->fetchModThumbnail($command, $curl, $mod->fileID, $mod->iconHash, $mod->iconExtension), 'license_id' => License::whereHubId($mod->licenseID)->value('id'), 'source_code_link' => $optionSourceCode?->source_code_link ?? '', 'featured' => (bool) $mod->isFeatured, 'contains_ai_content' => (bool) $optionContainsAi?->contains_ai ?? false, 'contains_ads' => (bool) $optionContainsAds?->contains_ads ?? false, 'disabled' => (bool) $mod->isDisabled, 'created_at' => Carbon::parse($mod->time, 'UTC'), 'updated_at' => Carbon::parse($mod->lastChangeTime, 'UTC'), ]; } if (! empty($insertData)) { Mod::upsert($insertData, ['hub_id'], ['user_id', 'name', 'slug', 'teaser', 'description', 'thumbnail', 'license_id', 'source_code_link', 'featured', 'contains_ai_content', 'disabled', 'created_at', 'updated_at']); $totalInserted += count($insertData); $command->line('Processed '.count($insertData).' mods. Total processed so far: '.$totalInserted); } unset($insertData); unset($mods); }, 'fileID'); curl_close($curl); $this->info('Total mods processed: '.$totalInserted); } protected function bringFileOptionsLocal(): void { DB::statement('CREATE TEMPORARY TABLE temp_file_option_values ( fileID INT, optionID INT, optionValue VARCHAR(255) )'); DB::connection('mysql_hub') ->table('filebase1_file_option_value') ->orderBy('fileID') ->chunk(200, function ($options) { foreach ($options as $option) { DB::table('temp_file_option_values')->insert([ 'fileID' => $option->fileID, 'optionID' => $option->optionID, 'optionValue' => $option->optionValue, ]); } }); } protected function bringFileContentLocal(): void { DB::statement('CREATE TEMPORARY TABLE temp_file_content ( fileID INT, subject VARCHAR(255), teaser VARCHAR(255), message LONGTEXT )'); DB::connection('mysql_hub') ->table('filebase1_file_content') ->orderBy('fileID') ->chunk(200, function ($contents) { foreach ($contents as $content) { DB::table('temp_file_content')->insert([ 'fileID' => $content->fileID, 'subject' => $content->subject, 'teaser' => $content->teaser, 'message' => $content->message, ]); } }); } protected function fetchModThumbnail($command, $curl, string $fileID, string $thumbnailHash, string $thumbnailExtension): string { if (empty($fileID) || empty($thumbnailHash) || empty($thumbnailExtension)) { return ''; } // Only the first two characters of the icon hash. $hashShort = substr($thumbnailHash, 0, 2); $hubUrl = "https://hub.sp-tarkov.com/files/images/file/$hashShort/$fileID.$thumbnailExtension"; $relativePath = "mods/$thumbnailHash.$thumbnailExtension"; $disk = match (config('app.env')) { 'production' => 'r2', default => 'local', }; // Check to make sure the image doesn't already exist. if (Storage::disk($disk)->exists($relativePath)) { return $relativePath; } $command->output->write("Downloading mod thumbnail: $hubUrl... "); curl_setopt($curl, CURLOPT_URL, $hubUrl); $image = curl_exec($curl); if ($image === false) { $command->error('Error: '.curl_error($curl)); } else { Storage::disk($disk)->put($relativePath, $image); $command->info('Done.'); } return $relativePath; } protected function importModVersions(): void { $command = $this; $totalInserted = 0; DB::connection('mysql_hub') ->table('filebase1_file_version') ->chunkById(500, function (Collection $versions) use (&$command, &$totalInserted) { foreach ($versions as $version) { $versionContent = DB::table('temp_file_version_content') ->select('description') ->where('versionID', $version->versionID) ->orderBy('versionID', 'desc') ->first(); $optionVirusTotal = DB::table('temp_file_option_values') ->select('optionValue as virus_total_link') ->where('fileID', $version->fileID) ->whereIn('optionID', [6, 2]) ->whereNot('optionValue', '') ->orderByDesc('optionID') ->first(); $versionLabel = DB::table('temp_file_version_labels') ->select('labelID') ->where('objectID', $version->fileID) ->orderBy('labelID', 'desc') ->first(); $modId = Mod::whereHubId($version->fileID)->value('id'); if (empty($versionLabel) || empty($modId)) { continue; } $insertData[] = [ 'hub_id' => $version->versionID, 'mod_id' => $modId, 'version' => $version->versionNumber, 'description' => $this->convertModDescription($versionContent->description ?? ''), 'link' => $version->downloadURL, 'spt_version_id' => SptVersion::whereHubId($versionLabel->labelID)->value('id'), 'virus_total_link' => $optionVirusTotal?->virus_total_link ?? '', 'downloads' => max((int) $version->downloads, 0), // Ensure the value is at least 0 'disabled' => (bool) $version->isDisabled, 'created_at' => Carbon::parse($version->uploadTime, 'UTC'), 'updated_at' => Carbon::parse($version->uploadTime, 'UTC'), ]; } if (! empty($insertData)) { ModVersion::upsert($insertData, ['hub_id'], ['mod_id', 'version', 'description', 'link', 'spt_version_id', 'virus_total_link', 'downloads', 'created_at', 'updated_at']); $totalInserted += count($insertData); $command->line('Processed '.count($insertData).' mod versions. Total processed so far: '.$totalInserted); } unset($insertData); unset($version); }, 'versionID'); $this->info('Total mod versions processed: '.$totalInserted); } protected function bringFileVersionLabelsLocal(): void { DB::statement('CREATE TEMPORARY TABLE temp_file_version_labels ( labelID INT, objectID INT )'); DB::connection('mysql_hub') ->table('wcf1_label_object') ->where('objectTypeID', 387) ->orderBy('labelID') ->chunk(200, function ($options) { foreach ($options as $option) { DB::table('temp_file_version_labels')->insert([ 'labelID' => $option->labelID, 'objectID' => $option->objectID, ]); } }); } protected function bringFileVersionContentLocal(): void { DB::statement('CREATE TEMPORARY TABLE temp_file_version_content ( versionID INT, description TEXT )'); DB::connection('mysql_hub') ->table('filebase1_file_version_content') ->orderBy('versionID') ->chunk(200, function ($options) { foreach ($options as $option) { DB::table('temp_file_version_content')->insert([ 'versionID' => $option->versionID, 'description' => $option->description, ]); } }); } protected function convertModDescription(string $description): string { // Alright, hear me out... Shut up. $converter = new HtmlConverter(); return $converter->convert(Purify::clean($description)); } }