diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php index c702423166c..6a277c6cbcb 100644 --- a/core/CronArchive/QueueConsumer.php +++ b/core/CronArchive/QueueConsumer.php @@ -100,6 +100,11 @@ class QueueConsumer */ private $siteTimer; + /** + * @var string + */ + private $currentSiteArchivingStartTime; + public function __construct(LoggerInterface $logger, $websiteIdArchiveList, $countOfProcesses, $pid, Model $model, SegmentArchiving $segmentArchiving, CronArchive $cronArchive, RequestParser $cliMultiRequestParser, ArchiveFilter $archiveFilter = null) @@ -156,6 +161,8 @@ public function getNextArchivesToProcess() // NOTE: we do this on every site iteration so we don't end up processing say a single user entered invalidation, // and then stop until the next hour. $this->cronArchive->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain($this->idSite); + + $this->currentSiteArchivingStartTime = Date::now()->getDatetime(); } // we don't want to invalidate different periods together or segment archives w/ no-segment archives @@ -329,7 +336,7 @@ private function getNextInvalidatedArchive($idSite, $extraInvalidationsToIgnore) while ($iterations < 100) { $invalidationsToExclude = array_merge($this->invalidationsToExclude, $extraInvalidationsToIgnore); - $nextArchive = $this->model->getNextInvalidatedArchive($idSite, $invalidationsToExclude); + $nextArchive = $this->model->getNextInvalidatedArchive($idSite, $this->currentSiteArchivingStartTime, $invalidationsToExclude); if (empty($nextArchive)) { break; } diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 60079946235..6ba5c165cf9 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -705,19 +705,21 @@ public function isSimilarArchiveInProgress($invalidation) /** * Gets the next invalidated archive that should be archived in a table. * - * @param string[] $tables - * @param int $count + * @param int $idSite + * @param string $archivingStartTime + * @param int[]|null $idInvalidationsToExclude * @param bool $useLimit Whether to limit the result set to one result or not. Used in tests only. */ - public function getNextInvalidatedArchive($idSite, $idInvalidationsToExclude = null, $useLimit = true) + public function getNextInvalidatedArchive($idSite, $archivingStartTime, $idInvalidationsToExclude = null, $useLimit = true) { $table = Common::prefixTable('archive_invalidations'); $sql = "SELECT idinvalidation, idarchive, idsite, date1, date2, period, `name`, report FROM `$table` - WHERE idsite = ? AND status != ?"; + WHERE idsite = ? AND status != ? AND ts_invalidated <= ?"; $bind = [ $idSite, ArchiveInvalidator::INVALIDATION_STATUS_IN_PROGRESS, + $archivingStartTime, ]; if (!empty($idInvalidationsToExclude)) { diff --git a/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php b/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php index 450648e030a..8cc5cae3dbb 100644 --- a/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php +++ b/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php @@ -104,6 +104,9 @@ public function test_invalidateConsumeOrder() ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], + + // high ts_invalidated, should not be selected + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-01-01', 'date2' => '2018-01-31', 'period' => 3, 'report' => null, 'ts_invalidated' => Date::factory(time() + 300)->getDatetime()], ]; shuffle($invalidations); @@ -504,6 +507,8 @@ private function makeTestArchiveFilter($restrictToDateRange = null, $restrictToP private function insertInvalidations(array $invalidations) { + $now = Date::now()->getDatetime(); + $table = Common::prefixTable('archive_invalidations'); foreach ($invalidations as $inv) { $bind = [ @@ -513,10 +518,11 @@ private function insertInvalidations(array $invalidations) $inv['date1'], $inv['date2'], $inv['period'], + isset($inv['ts_invalidated']) ? $inv['ts_invalidated'] : $now, $inv['report'], ]; Db::query("INSERT INTO `$table` (idarchive, name, idsite, date1, date2, period, ts_invalidated, report, status) - VALUES (?, ?, ?, ?, ?, ?, NOW(), ?, 0)", $bind); + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)", $bind); } } diff --git a/tests/PHPUnit/Integration/DataAccess/ModelTest.php b/tests/PHPUnit/Integration/DataAccess/ModelTest.php index 15305796ee6..070e1c8f64a 100644 --- a/tests/PHPUnit/Integration/DataAccess/ModelTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ModelTest.php @@ -505,7 +505,7 @@ public function test_getNextInvalidatedArchive_returnsCorrectOrder() ), ); - $actual = $this->model->getNextInvalidatedArchive(1, null, $useLimit = false); + $actual = $this->model->getNextInvalidatedArchive(1, '2030-01-01 00:00:00', null, $useLimit = false); $this->assertEquals($expected, $actual); } @@ -547,11 +547,12 @@ private function insertArchiveData($archivesToInsert) private function insertInvalidations(array $invalidations) { $table = Common::prefixTable('archive_invalidations'); + $now = Date::now()->getDatetime(); foreach ($invalidations as $invalidation) { $sql = "INSERT INTO `$table` (idsite, date1, date2, period, `name`, status, ts_invalidated, ts_started) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; Db::query($sql, [ $invalidation['idsite'] ?? 1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name'], - $invalidation['status'] ?? 0, $invalidation['ts_invalidated'] ?? null, $invalidation['ts_started'] ?? null, + $invalidation['status'] ?? 0, $invalidation['ts_invalidated'] ?? $now, $invalidation['ts_started'] ?? null, ]); } }