bringFileAuthorsLocal(); $this->bringFileOptionsLocal(); $this->bringFileContentLocal(); $this->bringFileVersionLabelsLocal(); $this->bringFileVersionContentLocal(); // Begin to import the data into the permanent local database tables. $this->importUsers(); $this->importLicenses(); $this->importSptVersions(); $this->importMods(); $this->importModVersions(); // Ensure that we've disconnected from the Hub database, clearing temporary tables. DB::connection('mysql_hub')->disconnect(); // Re-sync search. Artisan::call('app:search-sync'); Artisan::call('cache:clear'); } /** * Bring the file authors from the Hub database to the local database temporary table. */ protected function bringFileAuthorsLocal(): void { DB::statement('DROP TEMPORARY TABLE IF EXISTS temp_file_author'); DB::statement('CREATE TEMPORARY TABLE temp_file_author ( fileID INT, userID INT )'); DB::connection('mysql_hub') ->table('filebase1_file_author') ->orderBy('fileID') ->chunk(200, function ($relationships) { foreach ($relationships as $relationship) { DB::table('temp_file_author')->insert([ 'fileID' => (int) $relationship->fileID, 'userID' => (int) $relationship->userID, ]); } }); } /** * Bring the file options from the Hub database to the local database temporary table. */ protected function bringFileOptionsLocal(): void { DB::statement('DROP TEMPORARY TABLE IF EXISTS temp_file_option_values'); DB::statement('CREATE TEMPORARY TABLE temp_file_option_values ( fileID INT, optionID INT, optionValue VARCHAR(255) )'); DB::connection('mysql_hub') ->table('filebase1_file_option_value') ->orderBy('fileID') ->chunk(200, function ($options) { foreach ($options as $option) { DB::table('temp_file_option_values')->insert([ 'fileID' => (int) $option->fileID, 'optionID' => (int) $option->optionID, 'optionValue' => $option->optionValue, ]); } }); } /** * Bring the file content from the Hub database to the local database temporary table. */ protected function bringFileContentLocal(): void { DB::statement('DROP TEMPORARY TABLE IF EXISTS temp_file_content'); DB::statement('CREATE TEMPORARY TABLE temp_file_content ( fileID INT, subject VARCHAR(255), teaser VARCHAR(255), message LONGTEXT )'); DB::connection('mysql_hub') ->table('filebase1_file_content') ->orderBy('fileID') ->chunk(200, function ($contents) { foreach ($contents as $content) { DB::table('temp_file_content')->insert([ 'fileID' => (int) $content->fileID, 'subject' => $content->subject, 'teaser' => $content->teaser, 'message' => $content->message, ]); } }); } /** * Bring the file version labels from the Hub database to the local database temporary table. */ protected function bringFileVersionLabelsLocal(): void { DB::statement('DROP TEMPORARY TABLE IF EXISTS temp_file_version_labels'); DB::statement('CREATE TEMPORARY TABLE temp_file_version_labels ( labelID INT, objectID INT )'); DB::connection('mysql_hub') ->table('wcf1_label_object') ->where('objectTypeID', 387) ->orderBy('labelID') ->chunk(200, function ($options) { foreach ($options as $option) { DB::table('temp_file_version_labels')->insert([ 'labelID' => (int) $option->labelID, 'objectID' => (int) $option->objectID, ]); } }); } /** * Bring the file version content from the Hub database to the local database temporary table. */ protected function bringFileVersionContentLocal(): void { DB::statement('DROP TEMPORARY TABLE IF EXISTS temp_file_version_content'); DB::statement('CREATE TEMPORARY TABLE temp_file_version_content ( versionID INT, description TEXT )'); DB::connection('mysql_hub') ->table('filebase1_file_version_content') ->orderBy('versionID') ->chunk(200, function ($options) { foreach ($options as $option) { DB::table('temp_file_version_content')->insert([ 'versionID' => (int) $option->versionID, 'description' => $option->description, ]); } }); } /** * Import the users from the Hub database to the local database. */ protected function importUsers(): void { DB::connection('mysql_hub') ->table('wcf1_user as u') ->select('u.userID', 'u.username', 'u.email', 'u.password', 'u.registrationDate', 'u.banned', 'u.banReason', 'u.banExpires', 'u.rankID', 'r.rankTitle') ->leftJoin('wcf1_user_rank as r', 'u.rankID', '=', 'r.rankID') ->chunkById(250, function (Collection $users) { $userData = $bannedUsers = $userRanks = []; foreach ($users as $user) { $userData[] = $this->collectUserData($user); $bannedUserData = $this->collectBannedUserData($user); if ($bannedUserData) { $bannedUsers[] = $bannedUserData; } $userRankData = $this->collectUserRankData($user); if ($userRankData) { $userRanks[] = $userRankData; } } $this->upsertUsers($userData); $this->handleBannedUsers($bannedUsers); $this->handleUserRoles($userRanks); }, 'userID'); } protected function collectUserData($user): array { return [ 'hub_id' => (int) $user->userID, 'name' => $user->username, 'email' => mb_convert_case($user->email, MB_CASE_LOWER, 'UTF-8'), 'password' => $this->cleanPasswordHash($user->password), 'created_at' => $this->cleanRegistrationDate($user->registrationDate), 'updated_at' => now('UTC')->toDateTimeString(), ]; } /** * Clean the password hash from the Hub database. */ protected function cleanPasswordHash(string $password): string { // The hub passwords sometimes hashed with a prefix of the hash type. We only want the hash. // If it's not Bcrypt, they'll have to reset their password. Tough luck. $clean = str_ireplace(['invalid:', 'bcrypt:', 'bcrypt::', 'cryptmd5:', 'cryptmd5::'], '', $password); // At this point, if the password hash starts with $2, it's a valid Bcrypt hash. Otherwise, it's invalid. return str_starts_with($clean, '$2') ? $clean : ''; } /** * Clean the registration date from the Hub database. */ protected function cleanRegistrationDate(string $registrationDate): string { $date = Carbon::createFromTimestamp($registrationDate); // If the registration date is in the future, set it to now. if ($date->isFuture()) { $date = Carbon::now('UTC'); } return $date->toDateTimeString(); } /** * Build an array of banned user data ready to be inserted into the local database. */ protected function collectBannedUserData($user): ?array { if ($user->banned) { return [ 'hub_id' => (int) $user->userID, 'comment' => $user->banReason ?? '', 'expired_at' => $this->cleanUnbannedAtDate($user->banExpires), ]; } return null; } /** * Clean the banned_at date from the Hub database. */ protected function cleanUnbannedAtDate(?string $unbannedAt): ?string { // If the input is null, return null if ($unbannedAt === null) { return null; } // Explicit check for the Unix epoch start date if (Str::contains($unbannedAt, '1970-01-01')) { return null; } // Use validator to check for a valid date format $validator = Validator::make(['date' => $unbannedAt], [ 'date' => 'date_format:Y-m-d H:i:s', ]); if ($validator->fails()) { // If the date format is invalid, return null return null; } // Validate the date using Carbon try { $date = Carbon::parse($unbannedAt); // Additional check to ensure the date is not a default or zero date if ($date->year == 0 || $date->month == 0 || $date->day == 0) { return null; } return $date->toDateTimeString(); } catch (\Exception $e) { // If the date is not valid, return null return null; } } /* * Build an array of user rank data ready to be inserted into the local database. */ protected function collectUserRankData($user): ?array { if ($user->rankID && $user->rankTitle) { return [ 'hub_id' => (int) $user->userID, 'title' => $user->rankTitle, ]; } return null; } /** * Insert or update the users in the local database. */ protected function upsertUsers($usersData): void { if (! empty($usersData)) { DB::table('users')->upsert($usersData, ['hub_id'], [ 'name', 'email', 'password', 'created_at', 'updated_at', ]); } } /** * Fetch the hub-banned users from the local database and ban them locally. */ protected function handleBannedUsers($bannedUsers): void { foreach ($bannedUsers as $bannedUser) { $user = User::whereHubId($bannedUser['hub_id'])->first(); $user->ban([ 'comment' => $bannedUser['comment'], 'expired_at' => $bannedUser['expired_at'], ]); } } /** * Fetch or create the user ranks in the local database and assign them to the users. */ protected function handleUserRoles($userRanks): void { foreach ($userRanks as $userRank) { $roleName = Str::ucfirst(Str::afterLast($userRank['title'], '.')); $roleData = $this->buildUserRoleData($roleName); UserRole::upsert($roleData, ['name'], ['name', 'short_name', 'description', 'color_class']); $userRole = UserRole::whereName($roleData['name'])->first(); $user = User::whereHubId($userRank['hub_id'])->first(); $user->assignRole($userRole); } } /** * Build the user role data based on the role name. */ protected function buildUserRoleData(string $name): array { if ($name === 'Administrator') { return [ 'name' => 'Administrator', 'short_name' => 'Admin', 'description' => 'An administrator has full access to the site.', 'color_class' => 'sky', ]; } if ($name === 'Moderator') { return [ 'name' => 'Moderator', 'short_name' => 'Mod', 'description' => 'A moderator has the ability to moderate user content.', 'color_class' => 'emerald', ]; } return [ 'name' => $name, 'short_name' => '', 'description' => '', 'color_class' => '', ]; } /** * Import the licenses from the Hub database to the local database. */ protected function importLicenses(): void { DB::connection('mysql_hub') ->table('filebase1_license') ->chunkById(100, function (Collection $licenses) { $insertData = []; foreach ($licenses as $license) { $insertData[] = [ 'hub_id' => (int) $license->licenseID, 'name' => $license->licenseName, 'link' => $license->licenseURL, ]; } if (! empty($insertData)) { DB::table('licenses')->upsert($insertData, ['hub_id'], ['name', 'link']); } }, 'licenseID'); } /** * Import the SPT versions from the Hub database to the local database. */ protected function importSptVersions(): void { DB::connection('mysql_hub') ->table('wcf1_label') ->where('groupID', 1) ->chunkById(100, function (Collection $versions) { $insertData = []; foreach ($versions as $version) { $insertData[] = [ 'hub_id' => (int) $version->labelID, 'version' => $version->label, 'color_class' => $this->translateColour($version->cssClassName), ]; } if (! empty($insertData)) { DB::table('spt_versions')->upsert($insertData, ['hub_id'], ['version', 'color_class']); } }, 'labelID'); } /** * Translate the colour class from the Hub database to the local database. */ protected function translateColour(string $colour = ''): string { return match ($colour) { 'green' => 'green', 'slightly-outdated' => 'lime', 'yellow' => 'yellow', 'red' => 'red', default => 'gray', }; } /** * Import the mods from the Hub database to the local database. */ protected function importMods(): void { // Initialize a cURL handler for downloading mod thumbnails. $curl = curl_init(); curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); DB::connection('mysql_hub') ->table('filebase1_file') ->chunkById(100, function (Collection $mods) use ($curl) { foreach ($mods as $mod) { // Fetch any additional authors for the mod. $modAuthors = DB::table('temp_file_author') ->where('fileID', $mod->fileID) ->pluck('userID') ->toArray(); $modAuthors[] = $mod->userID; // Add the primary author to the list. $modAuthors = User::whereIn('hub_id', $modAuthors)->pluck('id')->toArray(); // Replace with local IDs. $modContent = DB::table('temp_file_content') ->where('fileID', $mod->fileID) ->orderBy('fileID', 'desc') ->first(); $optionSourceCode = DB::table('temp_file_option_values') ->select('optionValue as source_code_link') ->where('fileID', $mod->fileID) ->whereIn('optionID', [5, 1]) ->whereNot('optionValue', '') ->orderByDesc('optionID') ->first(); $optionContainsAi = DB::table('temp_file_option_values') ->select('optionValue as contains_ai') ->where('fileID', $mod->fileID) ->where('optionID', 7) ->whereNot('optionValue', '') ->first(); $optionContainsAds = DB::table('temp_file_option_values') ->select('optionValue as contains_ads') ->where('fileID', $mod->fileID) ->where('optionID', 3) ->whereNot('optionValue', '') ->first(); $versionLabel = DB::table('temp_file_version_labels') ->select('labelID') ->where('objectID', $mod->fileID) ->orderBy('labelID', 'desc') ->first(); // Skip the mod if it doesn't have a version label attached to it. if (empty($versionLabel)) { continue; } $modData[] = [ 'hub_id' => (int) $mod->fileID, 'users' => $modAuthors, 'name' => $modContent?->subject ?? '', 'slug' => Str::slug($modContent?->subject ?? ''), 'teaser' => Str::limit($modContent?->teaser ?? ''), 'description' => $this->cleanHubContent($modContent?->message ?? ''), 'thumbnail' => $this->fetchModThumbnail($curl, $mod->fileID, $mod->iconHash, $mod->iconExtension), 'license_id' => License::whereHubId($mod->licenseID)->value('id'), 'source_code_link' => $optionSourceCode?->source_code_link ?? '', 'featured' => (bool) $mod->isFeatured, 'contains_ai_content' => (bool) $optionContainsAi?->contains_ai, 'contains_ads' => (bool) $optionContainsAds?->contains_ads, 'disabled' => (bool) $mod->isDisabled, 'published_at' => Carbon::parse($mod->time, 'UTC'), 'created_at' => Carbon::parse($mod->time, 'UTC'), 'updated_at' => Carbon::parse($mod->lastChangeTime, 'UTC'), ]; } if (! empty($modData)) { // Remove the user_id from the mod data before upserting. $insertModData = array_map(fn ($mod) => Arr::except($mod, 'users'), $modData); Mod::withoutGlobalScopes()->upsert($insertModData, ['hub_id'], [ 'name', 'slug', 'teaser', 'description', 'thumbnail', 'license_id', 'source_code_link', 'featured', 'contains_ai_content', 'contains_ads', 'disabled', 'published_at', 'created_at', 'updated_at', ]); foreach ($modData as $mod) { Mod::whereHubId($mod['hub_id'])->first()?->users()->sync($mod['users']); } } }, 'fileID'); // Close the cURL handler. curl_close($curl); } /** * Convert the mod description from WoltHub flavoured HTML to Markdown. */ protected function cleanHubContent(string $dirty): string { // Alright, hear me out... Shut up. $converter = new HtmlConverter; $clean = Purify::clean($dirty); return $converter->convert($clean); } /** * Fetch the mod thumbnail from the Hub and store it anew. */ protected function fetchModThumbnail(CurlHandle $curl, string $fileID, string $thumbnailHash, string $thumbnailExtension): string { // If any of the required fields are empty, return an empty string. if (empty($fileID) || empty($thumbnailHash) || empty($thumbnailExtension)) { return ''; } // Build some paths/URLs using the mod data. $hashShort = substr($thumbnailHash, 0, 2); $fileName = $fileID.'.'.$thumbnailExtension; $hubUrl = 'https://hub.sp-tarkov.com/files/images/file/'.$hashShort.'/'.$fileName; $relativePath = 'mods/'.$fileName; // Determine the disk to use based on the environment. $disk = match (config('app.env')) { 'production' => 'r2', // Cloudflare R2 Storage default => 'public', // Local }; // Check to make sure the image doesn't already exist. if (Storage::disk($disk)->exists($relativePath)) { return $relativePath; // Already exists, return the path. } // Download the image using the cURL handler. curl_setopt($curl, CURLOPT_URL, $hubUrl); $image = curl_exec($curl); if ($image === false) { Log::error('There was an error attempting to download a mod thumbnail. cURL error: '.curl_error($curl)); return ''; } // Store the image on the disk. Storage::disk($disk)->put($relativePath, $image); return $relativePath; } /** * Import the mod versions from the Hub database to the local database. */ protected function importModVersions(): void { DB::connection('mysql_hub') ->table('filebase1_file_version') ->chunkById(500, function (Collection $versions) { foreach ($versions as $version) { $versionContent = DB::table('temp_file_version_content') ->select('description') ->where('versionID', $version->versionID) ->orderBy('versionID', 'desc') ->first(); $optionVirusTotal = DB::table('temp_file_option_values') ->select('optionValue as virus_total_link') ->where('fileID', $version->fileID) ->whereIn('optionID', [6, 2]) ->whereNot('optionValue', '') ->orderByDesc('optionID') ->first(); $versionLabel = DB::table('temp_file_version_labels') ->select('labelID') ->where('objectID', $version->fileID) ->orderBy('labelID', 'desc') ->first(); $modId = Mod::whereHubId($version->fileID)->value('id'); // Skip the mod version if it doesn't have a mod or version label attached to it. if (empty($versionLabel) || empty($modId)) { continue; } $insertData[] = [ 'hub_id' => (int) $version->versionID, 'mod_id' => $modId, 'version' => $version->versionNumber, 'description' => $this->cleanHubContent($versionContent->description ?? ''), 'link' => $version->downloadURL, 'spt_version_id' => SptVersion::whereHubId($versionLabel->labelID)->value('id'), 'virus_total_link' => $optionVirusTotal?->virus_total_link ?? '', 'downloads' => max((int) $version->downloads, 0), // At least 0. 'disabled' => (bool) $version->isDisabled, 'published_at' => Carbon::parse($version->uploadTime, 'UTC'), 'created_at' => Carbon::parse($version->uploadTime, 'UTC'), 'updated_at' => Carbon::parse($version->uploadTime, 'UTC'), ]; } if (! empty($insertData)) { ModVersion::withoutGlobalScopes()->upsert($insertData, ['hub_id'], [ 'mod_id', 'version', 'description', 'link', 'spt_version_id', 'virus_total_link', 'downloads', 'published_at', 'created_at', 'updated_at', ]); } }, 'versionID'); } /** * The job failed to process. */ public function failed(Exception $exception): void { // Explicitly drop the temporary tables. DB::unprepared('DROP TEMPORARY TABLE IF EXISTS temp_file_author'); DB::unprepared('DROP TEMPORARY TABLE IF EXISTS temp_file_option_values'); DB::unprepared('DROP TEMPORARY TABLE IF EXISTS temp_file_content'); DB::unprepared('DROP TEMPORARY TABLE IF EXISTS temp_file_version_labels'); DB::unprepared('DROP TEMPORARY TABLE IF EXISTS temp_file_version_content'); // Close the connections. This should drop the temporary tables as well, but I like to be explicit. DB::connection('mysql_hub')->disconnect(); DB::disconnect(); } }