Query Optimization

- Download counts were taking too long to calculate dynamically, so we're keeping track of a total count with observers and queued job.
- Optimized the SQL used to order a mod listing by mod version update times.

app/Console/Commands/ImportHubCommand.php

@@ -9,12 +9,12 @@ class ImportHubCommand extends Command
 {
     protected $signature = 'app:import-hub';
 
-    protected $description = 'Connects to the Hub database and imports the data into the Laravel database.';
+    protected $description = 'Connects to the Hub database and imports the data into the Laravel database';
 
     public function handle(): void
     {
         ImportHubDataJob::dispatch()->onQueue('long');
 
-        $this->info('The import job has been added to the queue.');
+        $this->info('ImportHubDataJob has been added to the queue');
     }
 }

app/Console/Commands/ResolveVersionsCommand.php

@@ -10,13 +10,13 @@ class ResolveVersionsCommand extends Command
 {
     protected $signature = 'app:resolve-versions';
 
-    protected $description = 'Resolve SPT and dependency versions for all mods.';
+    protected $description = 'Resolve SPT and dependency versions for all mods';
 
     public function handle(): void
     {
         ResolveSptVersionsJob::dispatch()->onQueue('default');
         ResolveDependenciesJob::dispatch()->onQueue('default');
 
-        $this->info('The import job has been added to the queue.');
+        $this->info('ResolveSptVersionsJob and ResolveDependenciesJob have been added to the queue');
     }
 }

app/Console/Commands/SearchSyncCommand.php

@@ -9,7 +9,7 @@ class SearchSyncCommand extends Command
 {
     protected $signature = 'app:search-sync';
 
-    protected $description = 'Syncs all search settings and indexes with the database data.';
+    protected $description = 'Syncs all search settings and indexes with the database data';
 
     public function handle(): void
     {
@@ -18,6 +18,6 @@ class SearchSyncCommand extends Command
         Artisan::call('scout:import', ['model' => '\App\Models\Mod']);
         Artisan::call('scout:import', ['model' => '\App\Models\User']);
 
-        $this->info('The search synchronisation jobs have been added to the queue.');
+        $this->info('The search synchronisation jobs have been added to the queue');
     }
 }

app/Console/Commands/SptVersionModCountsCommand.php

@@ -9,12 +9,12 @@ class SptVersionModCountsCommand extends Command
 {
     protected $signature = 'app:count-mods';
 
-    protected $description = 'Recalculate the mod counts for each SPT version.';
+    protected $description = 'Recalculate the mod counts for each SPT version';
 
     public function handle(): void
     {
         SptVersionModCountsJob::dispatch()->onQueue('default');
 
-        $this->info('The count job has been added to the queue.');
+        $this->info('SptVersionModCountsJob has been added to the queue');
     }
 }

app/Console/Commands/UpdateModDownloadsCommand.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\UpdateModDownloadsJob;
+use Illuminate\Console\Command;
+
+class UpdateModDownloadsCommand extends Command
+{
+    protected $signature = 'app:update-downloads';
+
+    protected $description = 'Recalculate total downloads for all mods';
+
+    public function handle(): void
+    {
+        UpdateModDownloadsJob::dispatch()->onQueue('default');
+
+        $this->info('UpdateModDownloadsJob added to the queue');
+    }
+}

app/Http/Controllers/ModController.php

@@ -27,8 +27,7 @@ class ModController extends Controller
 
     public function show(int $modId, string $slug)
     {
-        $mod = Mod::withTotalDownloads()
+        $mod = Mod::with([
-            ->with([
             'versions',
             'versions.latestSptVersion:id,version,color_class',
             'versions.latestResolvedDependencies',

app/Http/Filters/ModFilter.php

@@ -5,6 +5,7 @@ namespace App\Http\Filters;
 use App\Models\Mod;
 use App\Models\ModVersion;
 use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Support\Facades\DB;
 
 class ModFilter
 {
@@ -36,10 +37,9 @@ class ModFilter
             'mods.teaser',
             'mods.thumbnail',
             'mods.featured',
+            'mods.downloads',
             'mods.created_at',
-        ])
+        ])->with([
-            ->withTotalDownloads()
-            ->with([
             'users:id,name',
             'latestVersion' => function ($query) {
                 $query->with('latestSptVersion:id,version,color_class');
@@ -70,17 +70,20 @@ class ModFilter
     {
         // We order the "recently updated" mods by the ModVersion's updated_at value.
         if ($type === 'updated') {
-            return $this->builder->orderByDesc(
+            return $this->builder
-                ModVersion::select('updated_at')
+                ->joinSub(
-                    ->whereColumn('mod_id', 'mods.id')
+                    ModVersion::select('mod_id', DB::raw('MAX(updated_at) as latest_updated_at'))->groupBy('mod_id'),
-                    ->orderByDesc('updated_at')
+                    'latest_versions',
-                    ->take(1)
+                    'mods.id',
-            );
+                    '=',
+                    'latest_versions.mod_id'
+                )
+                ->orderByDesc('latest_versions.latest_updated_at');
         }
 
         // By default, we simply order by the column on the mods table/query.
         $column = match ($type) {
-            'downloaded' => 'total_downloads',
+            'downloaded' => 'downloads',
             default => 'created_at',
         };
 

app/Jobs/ImportHubDataJob.php

@@ -57,6 +57,7 @@ class ImportHubDataJob implements ShouldBeUnique, ShouldQueue
         Artisan::call('app:search-sync');
         Artisan::call('app:resolve-versions');
         Artisan::call('app:count-mods');
+        Artisan::call('app:update-downloads');
         Artisan::call('cache:clear');
     }
 

app/Jobs/UpdateModDownloadsJob.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Mod;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class UpdateModDownloadsJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Recalculate the total download counts for each mod.
+     */
+    public function handle(): void
+    {
+        Mod::with('versions')->chunk(100, function ($mods) {
+            foreach ($mods as $mod) {
+                $mod->calculateDownloads();
+            }
+        });
+    }
+}

app/Models/Mod.php

@@ -39,6 +39,15 @@ class Mod extends Model
         static::addGlobalScope(new PublishedScope);
     }
 
+    /**
+     * Calculate the total number of downloads for the mod.
+     */
+    public function calculateDownloads(): void
+    {
+        $this->downloads = $this->versions->sum('downloads');
+        $this->saveQuietly();
+    }
+
     /**
      * The relationship between a mod and its users.
      */
@@ -61,28 +70,17 @@ class Mod extends Model
     public function versions(): HasMany
     {
         return $this->hasMany(ModVersion::class)
-            ->whereHas('sptVersions')
+            ->whereHas('latestSptVersion')
             ->orderByDesc('version');
     }
 
-    /**
-     * Scope a query to include the total number of downloads for a mod.
-     */
-    public function scopeWithTotalDownloads($query)
-    {
-        return $query->addSelect([
-            'total_downloads' => ModVersion::selectRaw('SUM(downloads) AS total_downloads')
-                ->whereColumn('mod_id', 'mods.id'),
-        ]);
-    }
-
     /**
      * The relationship between a mod and its last updated version.
      */
     public function lastUpdatedVersion(): HasOne
     {
         return $this->hasOne(ModVersion::class)
-            ->whereHas('sptVersions')
+            ->whereHas('latestSptVersion')
             ->orderByDesc('updated_at');
     }
 

app/Observers/ModVersionObserver.php

@@ -29,7 +29,8 @@ class ModVersionObserver
 
         $this->sptVersionService->resolve($modVersion);
 
-        $this->updateRelatedSptVersions($modVersion); // Always done after resolving SPT versions.
+        $this->updateRelatedSptVersions($modVersion); // After resolving SPT versions.
+        $this->updateRelatedMod($modVersion);
     }
 
     /**
@@ -44,6 +45,15 @@ class ModVersionObserver
         }
     }
 
+    /**
+     * Update properties on the related Mod.
+     */
+    protected function updateRelatedMod(ModVersion $modVersion): void
+    {
+        $mod = $modVersion->mod;
+        $mod->calculateDownloads();
+    }
+
     /**
      * Handle the ModVersion "deleted" event.
      */
@@ -51,6 +61,7 @@ class ModVersionObserver
     {
         $this->dependencyVersionService->resolve($modVersion);
 
-        $this->updateRelatedSptVersions($modVersion); // Always done after resolving SPT versions.
+        $this->updateRelatedSptVersions($modVersion); // After resolving SPT versions.
+        $this->updateRelatedMod($modVersion);
     }
 }

app/View/Components/ModListSection.php

@@ -6,6 +6,7 @@ use App\Models\Mod;
 use App\Models\ModVersion;
 use Illuminate\Contracts\View\View;
 use Illuminate\Database\Eloquent\Collection;
+use Illuminate\Support\Facades\DB;
 use Illuminate\View\Component;
 
 class ModListSection extends Component
@@ -25,24 +26,22 @@ class ModListSection extends Component
 
     private function fetchFeaturedMods(): Collection
     {
-        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured'])
+        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured', 'downloads'])
-            ->withTotalDownloads()
             ->with([
                 'latestVersion',
                 'latestVersion.latestSptVersion:id,version,color_class',
                 'users:id,name',
                 'license:id,name,link',
             ])
-            ->where('featured', true)
+            ->whereFeatured(true)
-            ->latest()
+            ->inRandomOrder()
             ->limit(6)
             ->get();
     }
 
     private function fetchLatestMods(): Collection
     {
-        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured', 'created_at'])
+        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured', 'created_at', 'downloads'])
-            ->withTotalDownloads()
             ->with([
                 'latestVersion',
                 'latestVersion.latestSptVersion:id,version,color_class',
@@ -56,20 +55,21 @@ class ModListSection extends Component
 
     private function fetchUpdatedMods(): Collection
     {
-        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured'])
+        return Mod::select(['id', 'name', 'slug', 'teaser', 'thumbnail', 'featured', 'downloads'])
-            ->withTotalDownloads()
             ->with([
-                'latestVersion',
+                'lastUpdatedVersion',
-                'latestVersion.latestSptVersion:id,version,color_class',
+                'lastUpdatedVersion.latestSptVersion:id,version,color_class',
                 'users:id,name',
                 'license:id,name,link',
             ])
-            ->orderByDesc(
+            ->joinSub(
-                ModVersion::select('updated_at')
+                ModVersion::select('mod_id', DB::raw('MAX(updated_at) as latest_updated_at'))->groupBy('mod_id'),
-                    ->whereColumn('mod_id', 'mods.id')
+                'latest_versions',
-                    ->orderByDesc('updated_at')
+                'mods.id',
-                    ->take(1)
+                '=',
+                'latest_versions.mod_id'
             )
+            ->orderByDesc('latest_versions.latest_updated_at')
             ->limit(6)
             ->get();
     }
@@ -85,17 +85,17 @@ class ModListSection extends Component
     {
         return [
             [
-                'title' => 'Featured Mods',
+                'title' => __('Featured Mods'),
                 'mods' => $this->modsFeatured,
                 'versionScope' => 'latestVersion',
             ],
             [
-                'title' => 'Newest Mods',
+                'title' => __('Newest Mods'),
                 'mods' => $this->modsLatest,
                 'versionScope' => 'latestVersion',
             ],
             [
-                'title' => 'Recently Updated Mods',
+                'title' => __('Recently Updated Mods'),
                 'mods' => $this->modsUpdated,
                 'versionScope' => 'lastUpdatedVersion',
             ],

database/migrations/2024_05_15_022710_create_mods_table.php

@@ -26,6 +26,7 @@ return new class extends Migration
                 ->constrained('licenses')
                 ->nullOnDelete()
                 ->cascadeOnUpdate();
+            $table->unsignedBigInteger('downloads')->default(0);
             $table->string('source_code_link');
             $table->boolean('featured')->default(false);
             $table->boolean('contains_ai_content')->default(false);

resources/views/components/mod-list-stats.blade.php

@@ -1,5 +1,5 @@
 <p {{ $attributes->class(['text-slate-700 dark:text-gray-300 text-sm']) }}>
-    <span title="{{ __('Exactly') }} {{ $mod->total_downloads }}">{{ Number::downloads($mod->total_downloads) }} downloads</span>
+    <span title="{{ __('Exactly') }} {{ $mod->downloads }}">{{ Number::downloads($mod->downloads) }} downloads</span>
     @if(!is_null($mod->created_at))
         <span>
             &mdash; Created

resources/views/mod/show.blade.php

@@ -38,7 +38,7 @@
                                 <a href="{{ $user->profileUrl() }}" class="text-slate-600 dark:text-gray-200 hover:underline">{{ $user->name }}</a>{{ $loop->last ? '' : ',' }}
                             @endforeach
                         </p>
-                        <p title="{{ __('Exactly') }} {{ $mod->total_downloads }}">{{ Number::downloads($mod->total_downloads) }} {{ __('Downloads') }}</p>
+                        <p title="{{ __('Exactly') }} {{ $mod->downloads }}">{{ Number::downloads($mod->downloads) }} {{ __('Downloads') }}</p>
                         <p class="mt-2">
                             <span class="badge-version {{ $mod->latestVersion->latestSptVersion->first()->color_class }} inline-flex items-center rounded-md px-2 py-1 text-xs font-medium text-nowrap">
                                 {{ $mod->latestVersion->latestSptVersion->first()->version_formatted }} {{ __('Compatible') }}

routes/console.php

@@ -3,10 +3,12 @@
 use App\Console\Commands\ImportHubCommand;
 use App\Console\Commands\ResolveVersionsCommand;
 use App\Console\Commands\SptVersionModCountsCommand;
+use App\Console\Commands\UpdateModDownloadsCommand;
 use Illuminate\Support\Facades\Schedule;
 
 Schedule::command(ImportHubCommand::class)->hourly();
 Schedule::command(ResolveVersionsCommand::class)->hourlyAt(30);
 Schedule::command(SptVersionModCountsCommand::class)->hourlyAt(40);
+Schedule::command(UpdateModDownloadsCommand::class)->hourlyAt(45);
 
 Schedule::command('horizon:snapshot')->everyFiveMinutes();