From 69839d12e375ead0830747ecf03466f6b33c45c0 Mon Sep 17 00:00:00 2001
From: Bozana Bokan <bozana.bokan@posteo.net>
Date: Thu, 6 Feb 2025 12:13:38 +0100
Subject: [PATCH] pkp/pkp-lib#9822 Increase timeout and improve DB table
 indexes for usage stats jobs

---
 .../migration/install/MetricsMigration.php    | 10 ++--
 ...ChangeUsageStatsTemporaryTablesIndexes.php | 56 +++++++++++++++++++
 dbscripts/xml/upgrade.xml                     |  1 +
 .../CompileUniqueInvestigations.php           |  2 +
 jobs/statistics/CompileUniqueRequests.php     |  2 +
 5 files changed, 66 insertions(+), 5 deletions(-)
 create mode 100644 classes/migration/upgrade/v3_4_0/I9822_ChangeUsageStatsTemporaryTablesIndexes.php

diff --git a/classes/migration/install/MetricsMigration.php b/classes/migration/install/MetricsMigration.php
index c918f97992a..2f7986abc39 100644
--- a/classes/migration/install/MetricsMigration.php
+++ b/classes/migration/install/MetricsMigration.php
@@ -320,7 +320,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'ust_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent', 'canonical_url'], 'ust_load_id_context_id_ip_ua_url');
         });
 
         // Usage stats unique book and chapter item investigations temporary records
@@ -361,7 +361,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'usii_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'usii_load_id_context_id_ip_ua');
         });
 
         // Usage stats unique book and chapter item requests temporary records
@@ -402,7 +402,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'usir_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'usir_load_id_context_id_ip_ua');
         });
 
         // Usage stats unique title investigations temporary records
@@ -442,7 +442,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'usti_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'usti_load_id_context_id_ip_ua');
         });
 
         // Usage stats unique title requests temporary records
@@ -483,7 +483,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'ustr_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'ustr_load_id_context_id_ip_ua');
         });
 
         // Usage stats institution temporary records
diff --git a/classes/migration/upgrade/v3_4_0/I9822_ChangeUsageStatsTemporaryTablesIndexes.php b/classes/migration/upgrade/v3_4_0/I9822_ChangeUsageStatsTemporaryTablesIndexes.php
new file mode 100644
index 00000000000..39b0e2ca7e2
--- /dev/null
+++ b/classes/migration/upgrade/v3_4_0/I9822_ChangeUsageStatsTemporaryTablesIndexes.php
@@ -0,0 +1,56 @@
+<?php
+
+/**
+ * @file classes/migration/upgrade/v3_4_0/I9822_ChangeUsageStatsTemporaryTablesIndexes.php
+ *
+ * Copyright (c) 2025 Simon Fraser University
+ * Copyright (c) 2025 John Willinsky
+ * Distributed under the GNU GPL v3. For full terms see the file docs/COPYING.
+ *
+ * @class I9822_ChangeUsageStatsTemporaryTablesIndexes
+ *
+ * @brief Consider aditional columns user_agent and canonical_url for the index on temporary usage stats tables to fix/improve the removeDoubleClicks and compileUniqueClicks query.
+ */
+
+namespace APP\migration\upgrade\v3_4_0;
+
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+use PKP\install\DowngradeNotSupportedException;
+
+class I9822_ChangeUsageStatsTemporaryTablesIndexes extends \PKP\migration\upgrade\v3_4_0\I9822_ChangeUsageStatsTemporaryTablesIndexes
+{
+    /**
+     * Run the migration.
+     */
+    public function up(): void
+    {
+        Schema::table('usage_stats_unique_title_investigations_temporary_records', function (Blueprint $table) {
+            if (Schema::hasIndex('usage_stats_unique_title_investigations_temporary_records', 'usti_load_id_context_id_ip')) {
+                $table->dropIndex('usti_load_id_context_id_ip');
+            }
+            if (!Schema::hasIndex('usage_stats_unique_title_investigations_temporary_records', 'usti_load_id_context_id_ip_ua')) {
+                $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'usti_load_id_context_id_ip_ua');
+            }
+        });
+        Schema::table('usage_stats_unique_title_requests_temporary_records', function (Blueprint $table) {
+            if (Schema::hasIndex('usage_stats_unique_title_requests_temporary_records', 'ustr_load_id_context_id_ip')) {
+                $table->dropIndex('ustr_load_id_context_id_ip');
+            }
+            if (!Schema::hasIndex('usage_stats_unique_title_requests_temporary_records', 'ustr_load_id_context_id_ip_ua')) {
+                $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'ustr_load_id_context_id_ip_ua');
+            }
+        });
+        parent::up();
+    }
+
+    /**
+     * Reverse the downgrades
+     *
+     * @throws DowngradeNotSupportedException
+     */
+    public function down(): void
+    {
+        throw new DowngradeNotSupportedException();
+    }
+}
diff --git a/dbscripts/xml/upgrade.xml b/dbscripts/xml/upgrade.xml
index 7b70c311aac..e302681bfa0 100644
--- a/dbscripts/xml/upgrade.xml
+++ b/dbscripts/xml/upgrade.xml
@@ -114,6 +114,7 @@
 	<upgrade minversion="3.4.0.0" maxversion="3.4.0.5">
 		<!-- Only new 3.4 installations need this fix -->
 		<migration class="PKP\migration\upgrade\v3_4_0\I8592_SiteNotificationSubscriptions"/>
+		<migration class="APP\migration\upgrade\v3_4_0\I9822_ChangeUsageStatsTemporaryTablesIndexes"/>
 	</upgrade>
 
 	<upgrade minversion="3.1.0.0" maxversion="3.4.9.9">
diff --git a/jobs/statistics/CompileUniqueInvestigations.php b/jobs/statistics/CompileUniqueInvestigations.php
index d748674c57f..8cf22e6929e 100644
--- a/jobs/statistics/CompileUniqueInvestigations.php
+++ b/jobs/statistics/CompileUniqueInvestigations.php
@@ -23,6 +23,8 @@
 
 class CompileUniqueInvestigations extends BaseJob
 {
+    public int $timeout = 600;
+
     /**
      * Create a new job instance.
      *
diff --git a/jobs/statistics/CompileUniqueRequests.php b/jobs/statistics/CompileUniqueRequests.php
index 2f37505b86c..1700ff8624d 100644
--- a/jobs/statistics/CompileUniqueRequests.php
+++ b/jobs/statistics/CompileUniqueRequests.php
@@ -23,6 +23,8 @@
 
 class CompileUniqueRequests extends BaseJob
 {
+    public int $timeout = 600;
+
     /**
      * Create a new job instance.
      *