Query Optimization

- Download counts were taking too long to calculate dynamically, so we're keeping track of a total count with observers and queued job. - Optimized the SQL used to order a mod listing by mod version update times.
2025-02-12 12:10:41 -05:00 · 2024-08-31 01:19:22 -04:00 · 2024-08-31 01:19:22 -04:00 · 16e3a67efd
commit 16e3a67efd
parent 09771d233a
16 changed files with 129 additions and 67 deletions
--- a/app/Console/Commands/ImportHubCommand.php
+++ b/app/Console/Commands/ImportHubCommand.php
@ -9,12 +9,12 @@ class ImportHubCommand extends Command
 {
    protected $signature = 'app:import-hub';

-    protected $description = 'Connects to the Hub database and imports the data into the Laravel database.';
+    protected $description = 'Connects to the Hub database and imports the data into the Laravel database';

    public function handle(): void
    {
        ImportHubDataJob::dispatch()->onQueue('long');

-        $this->info('The import job has been added to the queue.');
+        $this->info('ImportHubDataJob has been added to the queue');
    }
 }
--- a/app/Console/Commands/ResolveVersionsCommand.php
+++ b/app/Console/Commands/ResolveVersionsCommand.php
@ -10,13 +10,13 @@ class ResolveVersionsCommand extends Command
 {
    protected $signature = 'app:resolve-versions';

-    protected $description = 'Resolve SPT and dependency versions for all mods.';
+    protected $description = 'Resolve SPT and dependency versions for all mods';

    public function handle(): void
    {
        ResolveSptVersionsJob::dispatch()->onQueue('default');
        ResolveDependenciesJob::dispatch()->onQueue('default');

-        $this->info('The import job has been added to the queue.');
+        $this->info('ResolveSptVersionsJob and ResolveDependenciesJob have been added to the queue');
    }
 }
--- a/app/Console/Commands/SearchSyncCommand.php
+++ b/app/Console/Commands/SearchSyncCommand.php
@ -9,7 +9,7 @@ class SearchSyncCommand extends Command
 {
    protected $signature = 'app:search-sync';

-    protected $description = 'Syncs all search settings and indexes with the database data.';
+    protected $description = 'Syncs all search settings and indexes with the database data';

    public function handle(): void
    {
@ -18,6 +18,6 @@ class SearchSyncCommand extends Command
        Artisan::call('scout:import', ['model' => '\App\Models\Mod']);
        Artisan::call('scout:import', ['model' => '\App\Models\User']);

-        $this->info('The search synchronisation jobs have been added to the queue.');
+        $this->info('The search synchronisation jobs have been added to the queue');
    }
 }
--- a/app/Console/Commands/SptVersionModCountsCommand.php
+++ b/app/Console/Commands/SptVersionModCountsCommand.php
@ -9,12 +9,12 @@ class SptVersionModCountsCommand extends Command
 {
    protected $signature = 'app:count-mods';

-    protected $description = 'Recalculate the mod counts for each SPT version.';
+    protected $description = 'Recalculate the mod counts for each SPT version';

    public function handle(): void
    {
        SptVersionModCountsJob::dispatch()->onQueue('default');

-        $this->info('The count job has been added to the queue.');
+        $this->info('SptVersionModCountsJob has been added to the queue');
    }
 }
--- a/app/Console/Commands/UpdateModDownloadsCommand.php
+++ b/app/Console/Commands/UpdateModDownloadsCommand.php
@ -0,0 +1,20 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\UpdateModDownloadsJob;
+use Illuminate\Console\Command;
+
+class UpdateModDownloadsCommand extends Command
+{
+    protected $signature = 'app:update-downloads';
+
+    protected $description = 'Recalculate total downloads for all mods';
+
+    public function handle(): void
+    {
+        UpdateModDownloadsJob::dispatch()->onQueue('default');
+
+        $this->info('UpdateModDownloadsJob added to the queue');
+    }
+}
--- a/app/Http/Controllers/ModController.php
+++ b/app/Http/Controllers/ModController.php
@ -27,15 +27,14 @@ class ModController extends Controller

    public function show(int $modId, string $slug)
    {
-        $mod = Mod::withTotalDownloads()
-            ->with([
-                'versions',
-                'versions.latestSptVersion:id,version,color_class',
-                'versions.latestResolvedDependencies',
-                'versions.latestResolvedDependencies.mod:id,name,slug',
-                'users:id,name',
-                'license:id,name,link',
-            ])
+        $mod = Mod::with([
+            'versions',
+            'versions.latestSptVersion:id,version,color_class',
+            'versions.latestResolvedDependencies',
+            'versions.latestResolvedDependencies.mod:id,name,slug',
+            'users:id,name',
+            'license:id,name,link',
+        ])
            ->whereHas('latestVersion')
            ->findOrFail($modId);

--- a/app/Http/Filters/ModFilter.php
+++ b/app/Http/Filters/ModFilter.php
@ -5,6 +5,7 @@ namespace App\Http\Filters;
 use App\Models\Mod;
 use App\Models\ModVersion;
 use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Support\Facades\DB;

 class ModFilter
 {
@ -36,15 +37,14 @@ class ModFilter
            'mods.teaser',
            'mods.thumbnail',
            'mods.featured',
+            'mods.downloads',
            'mods.created_at',
-        ])
-            ->withTotalDownloads()
-            ->with([
-                'users:id,name',
-                'latestVersion' => function ($query) {
-                    $query->with('latestSptVersion:id,version,color_class');
-                },
-            ]);
+        ])->with([
+            'users:id,name',
+            'latestVersion' => function ($query) {
+                $query->with('latestSptVersion:id,version,color_class');
+            },
+        ]);
    }

    /**
@ -70,17 +70,20 @@ class ModFilter
    {
        // We order the "recently updated" mods by the ModVersion's updated_at value.
        if ($type === 'updated') {
-            return $this->builder->orderByDesc(
-                ModVersion::select('updated_at')
-                    ->whereColumn('mod_id', 'mods.id')
-                    ->orderByDesc('updated_at')
-                    ->take(1)
-            );
+            return $this->builder
+                ->joinSub(
+                    ModVersion::select('mod_id', DB::raw('MAX(updated_at) as latest_updated_at'))->groupBy('mod_id'),
+                    'latest_versions',
+                    'mods.id',
+                    '=',
+                    'latest_versions.mod_id'
+                )
+                ->orderByDesc('latest_versions.latest_updated_at');
        }

        // By default, we simply order by the column on the mods table/query.
        $column = match ($type) {
-            'downloaded' => 'total_downloads',
+            'downloaded' => 'downloads',
            default => 'created_at',
        };

--- a/app/Jobs/ImportHubDataJob.php
+++ b/app/Jobs/ImportHubDataJob.php
@ -57,6 +57,7 @@ class ImportHubDataJob implements ShouldBeUnique, ShouldQueue
        Artisan::call('app:search-sync');
        Artisan::call('app:resolve-versions');
        Artisan::call('app:count-mods');
+        Artisan::call('app:update-downloads');
        Artisan::call('cache:clear');
    }

--- a/app/Jobs/UpdateModDownloadsJob.php
+++ b/app/Jobs/UpdateModDownloadsJob.php
@ -0,0 +1,27 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Mod;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class UpdateModDownloadsJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Recalculate the total download counts for each mod.
+     */
+    public function handle(): void
+    {
+        Mod::with('versions')->chunk(100, function ($mods) {
+            foreach ($mods as $mod) {
+                $mod->calculateDownloads();
+            }
+        });
+    }
+}
--- a/app/Models/Mod.php
+++ b/app/Models/Mod.php
@ -39,6 +39,15 @@ class Mod extends Model
        static::addGlobalScope(new PublishedScope);
    }

+    /**
+     * Calculate the total number of downloads for the mod.
+     */
+    public function calculateDownloads(): void
+    {
+        $this->downloads = $this->versions->sum('downloads');
+        $this->saveQuietly();
+    }
+
    /**
     * The relationship between a mod and its users.
     */
@ -61,28 +70,17 @@ class Mod extends Model
    public function versions(): HasMany
    {
        return $this->hasMany(ModVersion::class)
-            ->whereHas('sptVersions')
+            ->whereHas('latestSptVersion')
            ->orderByDesc('version');
    }

-    /**
-     * Scope a query to include the total number of downloads for a mod.
-     */
-    public function scopeWithTotalDownloads($query)
-    {
-        return $query->addSelect([
-            'total_downloads' => ModVersion::selectRaw('SUM(downloads) AS total_downloads')
-                ->whereColumn('mod_id', 'mods.id'),
-        ]);
-    }
-
    /**
     * The relationship between a mod and its last updated version.
     */
    public function lastUpdatedVersion(): HasOne
    {
        return $this->hasOne(ModVersion::class)
-            ->whereHas('sptVersions')
+            ->whereHas('latestSptVersion')
            ->orderByDesc('updated_at');
    }

--- a/app/Observers/ModVersionObserver.php
+++ b/app/Observers/ModVersionObserver.php
@ -29,7 +29,8 @@ class ModVersionObserver

        $this->sptVersionService->resolve($modVersion);

-        $this->updateRelatedSptVersions($modVersion); // Always done after resolving SPT versions.
+        $this->updateRelatedSptVersions($modVersion); // After resolving SPT versions.
+        $this->updateRelatedMod($modVersion);
    }

    /**
@ -44,6 +45,15 @@ class ModVersionObserver
        }
    }

+    /**
+     * Update properties on the related Mod.
+     */
+    protected function updateRelatedMod(ModVersion $modVersion): void
+    {
+        $mod = $modVersion->mod;
+        $mod->calculateDownloads();
+    }
+
    /**
     * Handle the ModVersion "deleted" event.
     */
@ -51,6 +61,7 @@ class ModVersionObserver
    {
        $this->dependencyVersionService->resolve($modVersion);

-        $this->updateRelatedSptVersions($modVersion); // Always done after resolving SPT versions.
+        $this->updateRelatedSptVersions($modVersion); // After resolving SPT versions.
+        $this->updateRelatedMod($modVersion);
    }
 }
--- a/app/View/Components/ModListSection.php
+++ b/app/View/Components/ModListSection.php
@ -6,6 +6,7 @@ use App\Models\Mod;
 use App\Models\ModVersion;
 use Illuminate\Contracts\View\View;
 use Illuminate\Database\Eloquent\Collection;
+use Illuminate\Support\Facades\DB;
 use Illuminate\View\Component;

 class ModListSection extends Component
@ -25,24 +26,22 @@ class ModListSection extends Component

    private function fetchFeaturedMods(): Collection
    {
-        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured'])
-            ->withTotalDownloads()
+        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured', 'downloads'])
            ->with([
                'latestVersion',
                'latestVersion.latestSptVersion:id,version,color_class',
                'users:id,name',
                'license:id,name,link',
            ])
-            ->where('featured', true)
-            ->latest()
+            ->whereFeatured(true)
+            ->inRandomOrder()
            ->limit(6)
            ->get();
    }

    private function fetchLatestMods(): Collection
    {
-        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured', 'created_at'])
-            ->withTotalDownloads()
+        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured', 'created_at', 'downloads'])
            ->with([
                'latestVersion',
                'latestVersion.latestSptVersion:id,version,color_class',
@ -56,20 +55,21 @@ class ModListSection extends Component

    private function fetchUpdatedMods(): Collection
    {
-        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured'])
-            ->withTotalDownloads()
+        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured', 'downloads'])
            ->with([
-                'latestVersion',
-                'latestVersion.latestSptVersion:id,version,color_class',
+                'lastUpdatedVersion',
+                'lastUpdatedVersion.latestSptVersion:id,version,color_class',
                'users:id,name',
                'license:id,name,link',
            ])
-            ->orderByDesc(
-                ModVersion::select('updated_at')
-                    ->whereColumn('mod_id', 'mods.id')
-                    ->orderByDesc('updated_at')
-                    ->take(1)
+            ->joinSub(
+                ModVersion::select('mod_id', DB::raw('MAX(updated_at) as latest_updated_at'))->groupBy('mod_id'),
+                'latest_versions',
+                'mods.id',
+                '=',
+                'latest_versions.mod_id'
            )
+            ->orderByDesc('latest_versions.latest_updated_at')
            ->limit(6)
            ->get();
    }
@ -85,17 +85,17 @@ class ModListSection extends Component
    {
        return [
            [
-                'title' => 'Featured Mods',
+                'title' => __('Featured Mods'),
                'mods' => $this->modsFeatured,
                'versionScope' => 'latestVersion',
            ],
            [
-                'title' => 'Newest Mods',
+                'title' => __('Newest Mods'),
                'mods' => $this->modsLatest,
                'versionScope' => 'latestVersion',
            ],
            [
-                'title' => 'Recently Updated Mods',
+                'title' => __('Recently Updated Mods'),
                'mods' => $this->modsUpdated,
                'versionScope' => 'lastUpdatedVersion',
            ],
--- a/database/migrations/2024_05_15_022710_create_mods_table.php
+++ b/database/migrations/2024_05_15_022710_create_mods_table.php
@ -26,6 +26,7 @@ return new class extends Migration
                ->constrained('licenses')
                ->nullOnDelete()
                ->cascadeOnUpdate();
+            $table->unsignedBigInteger('downloads')->default(0);
            $table->string('source_code_link');
            $table->boolean('featured')->default(false);
            $table->boolean('contains_ai_content')->default(false);
--- a/resources/views/components/mod-list-stats.blade.php
+++ b/resources/views/components/mod-list-stats.blade.php
@ -1,5 +1,5 @@
 <p {{ $attributes->class(['text-slate-700 dark:text-gray-300 text-sm']) }}>
-    <span title="{{ __('Exactly') }} {{ $mod->total_downloads }}">{{ Number::downloads($mod->total_downloads) }} downloads</span>
+    <span title="{{ __('Exactly') }} {{ $mod->downloads }}">{{ Number::downloads($mod->downloads) }} downloads</span>
    @if(!is_null($mod->created_at))
        <span>
            &mdash; Created
--- a/resources/views/mod/show.blade.php
+++ b/resources/views/mod/show.blade.php
@ -38,7 +38,7 @@
                                <a href="{{ $user->profileUrl() }}" class="text-slate-600 dark:text-gray-200 hover:underline">{{ $user->name }}</a>{{ $loop->last ? '' : ',' }}
                            @endforeach
                        </p>
-                        <p title="{{ __('Exactly') }} {{ $mod->total_downloads }}">{{ Number::downloads($mod->total_downloads) }} {{ __('Downloads') }}</p>
+                        <p title="{{ __('Exactly') }} {{ $mod->downloads }}">{{ Number::downloads($mod->downloads) }} {{ __('Downloads') }}</p>
                        <p class="mt-2">
                            <span class="badge-version {{ $mod->latestVersion->latestSptVersion->first()->color_class }} inline-flex items-center rounded-md px-2 py-1 text-xs font-medium text-nowrap">
                                {{ $mod->latestVersion->latestSptVersion->first()->version_formatted }} {{ __('Compatible') }}
--- a/routes/console.php
+++ b/routes/console.php
@ -3,10 +3,12 @@
 use App\Console\Commands\ImportHubCommand;
 use App\Console\Commands\ResolveVersionsCommand;
 use App\Console\Commands\SptVersionModCountsCommand;
+use App\Console\Commands\UpdateModDownloadsCommand;
 use Illuminate\Support\Facades\Schedule;

 Schedule::command(ImportHubCommand::class)->hourly();
 Schedule::command(ResolveVersionsCommand::class)->hourlyAt(30);
 Schedule::command(SptVersionModCountsCommand::class)->hourlyAt(40);
+Schedule::command(UpdateModDownloadsCommand::class)->hourlyAt(45);

 Schedule::command('horizon:snapshot')->everyFiveMinutes();