From 1cb376c5734457b5ee90805aa1102ae8c7be3d32 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Sun, 1 Mar 2015 15:09:58 -0800 Subject: [PATCH 01/31] Refs #7181, change ArchivePurger to use instance methods instead of static methods, move Rules::shouldPurgeOutdatedArchives since it is only used by ArchivePurger and move comment in said function. --- core/ArchiveProcessor/Rules.php | 45 --------------- core/DataAccess/ArchivePurger.php | 93 ++++++++++++++++++++++++------- plugins/CoreAdminHome/Tasks.php | 14 ++++- 3 files changed, 84 insertions(+), 68 deletions(-) diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php index 1e94a046dba..7d4ac8e5648 100644 --- a/core/ArchiveProcessor/Rules.php +++ b/core/ArchiveProcessor/Rules.php @@ -131,51 +131,6 @@ public static function getDoneFlags(array $plugins, Segment $segment, $isSkipAgg return $doneFlags; } - /** - * Returns false if we should not purge data for this month, - * or returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged. - * - * Note: when calling this function it is assumed that the callee will purge the outdated archives afterwards. - * - * @param \Piwik\Date $date - * @return int|bool Outdated archives older than this timestamp should be purged - */ - public static function shouldPurgeOutdatedArchives(Date $date) - { - if (! self::isRequestAuthorizedToArchive()){ - Log::info("Purging temporary archives: skipped (no authorization)"); - return false; - } - - $key = self::FLAG_TABLE_PURGED . "blob_" . $date->toString('Y_m'); - $timestamp = Option::get($key); - - // we shall purge temporary archives after their timeout is finished, plus an extra 6 hours - // in case archiving is disabled or run once a day, we give it this extra time to run - // and re-process more recent records... - $temporaryArchivingTimeout = self::getTodayArchiveTimeToLive(); - $hoursBetweenPurge = 6; - $purgeEveryNSeconds = max($temporaryArchivingTimeout, $hoursBetweenPurge * 3600); - - // we only delete archives if we are able to process them, otherwise, the browser might process reports - // when &segment= is specified (or custom date range) and would below, delete temporary archives that the - // browser is not able to process until next cron run (which could be more than 1 hour away) - if ($timestamp !== false && $timestamp >= time() - $purgeEveryNSeconds) { - Log::info("Purging temporary archives: skipped (purging every " . $hoursBetweenPurge . "hours)"); - return false; - } - - Option::set($key, time()); - - if (self::isBrowserTriggerEnabled()) { - // If Browser Archiving is enabled, it is likely there are many more temporary archives - // We delete more often which is safe, since reports are re-processed on demand - return Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime(); - } - - // If cron core:archive command is building the reports, we should keep all temporary reports from today - return Date::factory('yesterday')->getDateTime(); - } public static function getMinTimeProcessedForTemporaryArchive( Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site) diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php index e7f185f4751..2bd544930ba 100644 --- a/core/DataAccess/ArchivePurger.php +++ b/core/DataAccess/ArchivePurger.php @@ -8,12 +8,12 @@ */ namespace Piwik\DataAccess; -use Exception; use Piwik\ArchiveProcessor\Rules; use Piwik\Config; use Piwik\Date; use Piwik\Db; use Piwik\Log; +use Piwik\Option; use Piwik\Piwik; /** @@ -29,7 +29,64 @@ */ class ArchivePurger { - public static function purgeInvalidatedArchives() + /** + * @var Model + */ + private $model; + + public function __construct(Model $model = null) + { + $this->model = $model ?: new Model(); + } + + /** + * Returns false if we should not purge data for this month, + * or returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged. + * + * Note: when calling this function it is assumed that the callee will purge the outdated archives afterwards. + * + * @param \Piwik\Date $date + * @return int|bool Outdated archives older than this timestamp should be purged + */ + public static function shouldPurgeOutdatedArchives(Date $date) + { + // we only delete archives if we are able to process them, otherwise, the browser might process reports + // when &segment= is specified (or custom date range) and would below, delete temporary archives that the + // browser is not able to process until next cron run (which could be more than 1 hour away) + if (!Rules::isRequestAuthorizedToArchive()) { + Log::info("Purging temporary archives: skipped (request not allowed to initiate archiving)"); + return false; + } + + $key = Rules::FLAG_TABLE_PURGED . "blob_" . $date->toString('Y_m'); + $timestamp = Option::get($key); + + // we shall purge temporary archives after their timeout is finished, plus an extra 6 hours + // in case archiving is disabled or run once a day, we give it this extra time to run + // and re-process more recent records... + $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive(); + $hoursBetweenPurge = 6; + $purgeEveryNSeconds = max($temporaryArchivingTimeout, $hoursBetweenPurge * 3600); + + if ($timestamp !== false && $timestamp >= time() - $purgeEveryNSeconds) { + Log::info("Purging temporary archives: skipped (purging every " . $hoursBetweenPurge . "hours)"); + return false; + } + + Option::set($key, time()); + + $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive(); + if (Rules::isBrowserTriggerEnabled()) { + // If Browser Archiving is enabled, it is likely there are many more temporary archives + // We delete more often which is safe, since reports are re-processed on demand + return Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime(); + } + + // If cron core:archive command is building the reports, we should keep all temporary reports from today + return Date::factory('yesterday')->getDateTime(); + } + + public function purgeInvalidatedArchives() { $store = new InvalidatedReports(); $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge(); @@ -41,12 +98,12 @@ public static function purgeInvalidatedArchives() $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); $numericTable = ArchiveTableCreator::getNumericTable($date); - $archiveIds = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites); + $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites); if (count($archiveIds) == 0) { continue; } - self::deleteArchiveIds($date, $archiveIds); + $this->deleteArchiveIds($date, $archiveIds); $store->markSiteIdsHaveBeenPurged($idSites, $yearMonth); } @@ -58,21 +115,21 @@ public static function purgeInvalidatedArchives() * * @param Date $dateStart Only the month will be used */ - public static function purgeOutdatedArchives(Date $dateStart) + public function purgeOutdatedArchives(Date $dateStart) { - $purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart); + $purgeArchivesOlderThan = self::shouldPurgeOutdatedArchives($dateStart); if (!$purgeArchivesOlderThan) { return; } - $idArchivesToDelete = self::getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan); + $idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan); if (!empty($idArchivesToDelete)) { - self::deleteArchiveIds($dateStart, $idArchivesToDelete); + $this->deleteArchiveIds($dateStart, $idArchivesToDelete); } - self::deleteArchivesWithPeriodRange($dateStart); + $this->deleteArchivesWithPeriodRange($dateStart); Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]", $purgeArchivesOlderThan, @@ -80,11 +137,11 @@ public static function purgeOutdatedArchives(Date $dateStart) implode(',', $idArchivesToDelete)); } - protected static function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan) + protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan) { $archiveTable = ArchiveTableCreator::getNumericTable($date); - $result = self::getModel()->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan); + $result = $this->model->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan); $idArchivesToDelete = array(); if (!empty($result)) { @@ -101,14 +158,14 @@ protected static function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderT * * @param $date Date */ - protected static function deleteArchivesWithPeriodRange(Date $date) + protected function deleteArchivesWithPeriodRange(Date $date) { $numericTable = ArchiveTableCreator::getNumericTable($date); $blobTable = ArchiveTableCreator::getBlobTable($date); $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days']; $pastDate = Date::factory('today')->subDay($daysRangesValid)->getDateTime(); - self::getModel()->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $pastDate); + $this->model->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $pastDate); Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]", $pastDate, $numericTable); @@ -120,20 +177,14 @@ protected static function deleteArchivesWithPeriodRange(Date $date) * @param Date $date * @param $idArchivesToDelete */ - protected static function deleteArchiveIds(Date $date, $idArchivesToDelete) + protected function deleteArchiveIds(Date $date, $idArchivesToDelete) { $batches = array_chunk($idArchivesToDelete, 1000); $numericTable = ArchiveTableCreator::getNumericTable($date); $blobTable = ArchiveTableCreator::getBlobTable($date); foreach ($batches as $idsToDelete) { - self::getModel()->deleteArchiveIds($numericTable, $blobTable, $idsToDelete); + $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete); } } - - private static function getModel() - { - return new Model(); - } - } diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index d633b9fd5d6..8ee38a433ab 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -15,6 +15,16 @@ class Tasks extends \Piwik\Plugin\Tasks { + /** + * @var ArchivePurger + */ + private $archivePurger; + + public function __construct(ArchivePurger $archivePurger = null) + { + $this->archivePurger = $archivePurger ?: new ArchivePurger(); + } + public function schedule() { // general data purge on older archive tables, executed daily @@ -36,14 +46,14 @@ public function purgeOutdatedArchives() // Somehow we may have archive tables created with older dates, prevent exception from being thrown if ($year > 1990) { - ArchivePurger::purgeOutdatedArchives(Date::factory("$year-$month-15")); + $this->archivePurger->purgeOutdatedArchives(Date::factory("$year-$month-15")); } } } public function purgeInvalidatedArchives() { - ArchivePurger::purgeInvalidatedArchives(); + $this->archivePurger->purgeInvalidatedArchives(); } public function optimizeArchiveTable() From 707dc96a77a352671e916e01fed420c92920e5dc Mon Sep 17 00:00:00 2001 From: diosmosis Date: Sun, 1 Mar 2015 15:28:06 -0800 Subject: [PATCH 02/31] Refs #7181, archive purging is only done by daily scheduled task (either through tracking or through CronArchive), so remove 6 hour forced time between purges in ArchivePurger::shouldPurgeOutdatedArchives. --- core/DataAccess/ArchivePurger.php | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php index 2bd544930ba..623071c84ab 100644 --- a/core/DataAccess/ArchivePurger.php +++ b/core/DataAccess/ArchivePurger.php @@ -58,23 +58,6 @@ public static function shouldPurgeOutdatedArchives(Date $date) return false; } - $key = Rules::FLAG_TABLE_PURGED . "blob_" . $date->toString('Y_m'); - $timestamp = Option::get($key); - - // we shall purge temporary archives after their timeout is finished, plus an extra 6 hours - // in case archiving is disabled or run once a day, we give it this extra time to run - // and re-process more recent records... - $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive(); - $hoursBetweenPurge = 6; - $purgeEveryNSeconds = max($temporaryArchivingTimeout, $hoursBetweenPurge * 3600); - - if ($timestamp !== false && $timestamp >= time() - $purgeEveryNSeconds) { - Log::info("Purging temporary archives: skipped (purging every " . $hoursBetweenPurge . "hours)"); - return false; - } - - Option::set($key, time()); - $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive(); if (Rules::isBrowserTriggerEnabled()) { // If Browser Archiving is enabled, it is likely there are many more temporary archives From a1fb019c795949faf5d1cb677f338842d6063ffc Mon Sep 17 00:00:00 2001 From: diosmosis Date: Sun, 1 Mar 2015 16:43:02 -0800 Subject: [PATCH 03/31] Refs #7181, remove isRequestAuthorizedToArchive call in Rules::shouldPurgeOutdatedArchives() so archive purging can be forced in contexts other than scheduled task running. --- core/DataAccess/ArchivePurger.php | 8 -------- plugins/CoreAdminHome/Tasks.php | 11 +++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php index 623071c84ab..cd8241b7972 100644 --- a/core/DataAccess/ArchivePurger.php +++ b/core/DataAccess/ArchivePurger.php @@ -50,14 +50,6 @@ public function __construct(Model $model = null) */ public static function shouldPurgeOutdatedArchives(Date $date) { - // we only delete archives if we are able to process them, otherwise, the browser might process reports - // when &segment= is specified (or custom date range) and would below, delete temporary archives that the - // browser is not able to process until next cron run (which could be more than 1 hour away) - if (!Rules::isRequestAuthorizedToArchive()) { - Log::info("Purging temporary archives: skipped (request not allowed to initiate archiving)"); - return false; - } - $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive(); if (Rules::isBrowserTriggerEnabled()) { // If Browser Archiving is enabled, it is likely there are many more temporary archives diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 8ee38a433ab..9682b502833 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -8,10 +8,12 @@ */ namespace Piwik\Plugins\CoreAdminHome; +use Piwik\ArchiveProcessor\Rules; use Piwik\DataAccess\ArchivePurger; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; +use Piwik\Log; class Tasks extends \Piwik\Plugin\Tasks { @@ -39,6 +41,15 @@ public function schedule() public function purgeOutdatedArchives() { + // TODO: is this correct? wouldn't segment archives create DONE archives, not DONE_TEMPORARY? should try to replicate in tests + // we only delete archives if we are able to process them, otherwise, the browser might process reports + // when &segment= is specified (or custom date range) and would below, delete temporary archives that the + // browser is not able to process until next cron run (which could be more than 1 hour away) + if (!Rules::isRequestAuthorizedToArchive()) { + Log::info("Purging temporary archives: skipped (request not allowed to initiate archiving)"); + return false; + } + $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled(); foreach ($archiveTables as $table) { $date = ArchiveTableCreator::getDateFromTableName($table); From 41b478a221e04e85db68f1a48139c1f802bb8672 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Mon, 2 Mar 2015 17:53:01 -0800 Subject: [PATCH 04/31] Move ArchivePurger class from core\DataAccess to core\Archive and rename Purger. --- .../ArchivePurger.php => Archive/Purger.php} | 57 +++++++++---------- core/DataAccess/ArchiveInvalidator.php | 2 +- plugins/CoreAdminHome/Tasks.php | 8 +-- plugins/TreemapVisualization | 2 +- tests/UI/expected-ui-screenshots | 2 +- 5 files changed, 33 insertions(+), 38 deletions(-) rename core/{DataAccess/ArchivePurger.php => Archive/Purger.php} (89%) diff --git a/core/DataAccess/ArchivePurger.php b/core/Archive/Purger.php similarity index 89% rename from core/DataAccess/ArchivePurger.php rename to core/Archive/Purger.php index cd8241b7972..535c3053e23 100644 --- a/core/DataAccess/ArchivePurger.php +++ b/core/Archive/Purger.php @@ -6,14 +6,16 @@ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later * */ -namespace Piwik\DataAccess; +namespace Piwik\Archive; use Piwik\ArchiveProcessor\Rules; use Piwik\Config; +use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataAccess\InvalidatedReports; +use Piwik\DataAccess\Model; use Piwik\Date; use Piwik\Db; use Piwik\Log; -use Piwik\Option; use Piwik\Piwik; /** @@ -24,10 +26,8 @@ * * (2) Deletes outdated archives (the temporary or errored archives) * - * - * @package Piwik\DataAccess */ -class ArchivePurger +class Purger { /** * @var Model @@ -39,30 +39,9 @@ public function __construct(Model $model = null) $this->model = $model ?: new Model(); } - /** - * Returns false if we should not purge data for this month, - * or returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged. - * - * Note: when calling this function it is assumed that the callee will purge the outdated archives afterwards. - * - * @param \Piwik\Date $date - * @return int|bool Outdated archives older than this timestamp should be purged - */ - public static function shouldPurgeOutdatedArchives(Date $date) - { - $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive(); - if (Rules::isBrowserTriggerEnabled()) { - // If Browser Archiving is enabled, it is likely there are many more temporary archives - // We delete more often which is safe, since reports are re-processed on demand - return Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime(); - } - - // If cron core:archive command is building the reports, we should keep all temporary reports from today - return Date::factory('yesterday')->getDateTime(); - } - public function purgeInvalidatedArchives() { + // TODO: why does it only look in specified sites to purge instead of all sites? $store = new InvalidatedReports(); $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge(); foreach ($idSitesByYearMonth as $yearMonth => $idSites) { @@ -92,14 +71,12 @@ public function purgeInvalidatedArchives() */ public function purgeOutdatedArchives(Date $dateStart) { - $purgeArchivesOlderThan = self::shouldPurgeOutdatedArchives($dateStart); - + $purgeArchivesOlderThan = self::getOldestTemporaryArchiveToKeepThreshold(); if (!$purgeArchivesOlderThan) { return; } $idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan); - if (!empty($idArchivesToDelete)) { $this->deleteArchiveIds($dateStart, $idArchivesToDelete); } @@ -162,4 +139,22 @@ protected function deleteArchiveIds(Date $date, $idArchivesToDelete) $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete); } } -} + + /** + * Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged. + * + * @return int|bool Outdated archives older than this timestamp should be purged + */ + private static function getOldestTemporaryArchiveToKeepThreshold() + { + $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive(); + if (Rules::isBrowserTriggerEnabled()) { + // If Browser Archiving is enabled, it is likely there are many more temporary archives + // We delete more often which is safe, since reports are re-processed on demand + return Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime(); + } + + // If cron core:archive command is building the reports, we should keep all temporary reports from today + return Date::factory('yesterday')->getDateTime(); + } +} \ No newline at end of file diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php index 916d94991b8..cc0d7d0941d 100644 --- a/core/DataAccess/ArchiveInvalidator.php +++ b/core/DataAccess/ArchiveInvalidator.php @@ -23,7 +23,7 @@ * * Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive) * - * The invalidated archives will be deleted by ArchivePurger + * The invalidated archives will be deleted by Purger * * @package Piwik\DataAccess */ diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 9682b502833..50b547cb62d 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -9,7 +9,7 @@ namespace Piwik\Plugins\CoreAdminHome; use Piwik\ArchiveProcessor\Rules; -use Piwik\DataAccess\ArchivePurger; +use Piwik\Archive\Purger; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; @@ -18,13 +18,13 @@ class Tasks extends \Piwik\Plugin\Tasks { /** - * @var ArchivePurger + * @var Purger */ private $archivePurger; - public function __construct(ArchivePurger $archivePurger = null) + public function __construct(Purger $archivePurger = null) { - $this->archivePurger = $archivePurger ?: new ArchivePurger(); + $this->archivePurger = $archivePurger ?: new Purger(); } public function schedule() diff --git a/plugins/TreemapVisualization b/plugins/TreemapVisualization index 705a9503381..5dbb90f027b 160000 --- a/plugins/TreemapVisualization +++ b/plugins/TreemapVisualization @@ -1 +1 @@ -Subproject commit 705a95033819dd40f207f42141d4c711f4043b63 +Subproject commit 5dbb90f027b6d214aebfa4f222975d32e99d7838 diff --git a/tests/UI/expected-ui-screenshots b/tests/UI/expected-ui-screenshots index ad20ff37e79..a19b0967a0b 160000 --- a/tests/UI/expected-ui-screenshots +++ b/tests/UI/expected-ui-screenshots @@ -1 +1 @@ -Subproject commit ad20ff37e799248e5815c7ddaa5ee758e9cdda93 +Subproject commit a19b0967a0b59bdb51c42d3517015789d2aef401 From c58556c6bac856bb996c9d36c45dff2f1e278086 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Tue, 3 Mar 2015 01:37:26 -0800 Subject: [PATCH 05/31] Mild refactor to Archive\Purger and add (failing) integration test for Archive\Purger. --- core/Archive/Purger.php | 137 ++++-- .../Integration/Archive/PurgerTest.php | 429 ++++++++++++++++++ 2 files changed, 539 insertions(+), 27 deletions(-) create mode 100644 tests/PHPUnit/Integration/Archive/PurgerTest.php diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index 535c3053e23..0b85ff2a068 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -34,33 +34,85 @@ class Purger */ private $model; - public function __construct(Model $model = null) + /** + * TODO: what does this class even do? + * + * @var InvalidatedReports + */ + private $invalidatedReports; + + /** + * TODO + * + * @var Date + */ + private $purgeCustomRangesOlderThan; + + /** + * TODO + * + * @var Date + */ + private $yesterday; + + /** + * TODO + * + * @var $today + */ + private $today; + + /** + * TODO + * + * @var int + */ + private $now; + + public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null) { $this->model = $model ?: new Model(); + $this->invalidatedReports = new InvalidatedReports(); + + $this->purgeCustomRangesOlderThan = $purgeCustomRangesOlderThan ?: self::getDefaultCustomRangeToPurgeAgeThreshold(); + + $this->yesterday = Date::factory('yesterday'); + $this->today = Date::factory('today'); + $this->now = time(); } + /** + * TODO + */ public function purgeInvalidatedArchives() { // TODO: why does it only look in specified sites to purge instead of all sites? - $store = new InvalidatedReports(); - $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge(); + $idSitesByYearMonth = $this->invalidatedReports->getSitesByYearMonthArchiveToPurge(); foreach ($idSitesByYearMonth as $yearMonth => $idSites) { - if(empty($idSites)) { - continue; - } - - $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); - $numericTable = ArchiveTableCreator::getNumericTable($date); + $this->purgeInvalidatedArchivesFrom($yearMonth, $idSites); + } + } - $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites); + /** + * TODO + */ + public function purgeInvalidatedArchivesFrom($yearMonth, $idSites) + { + if (empty($idSites)) { + return; + } - if (count($archiveIds) == 0) { - continue; - } - $this->deleteArchiveIds($date, $archiveIds); + $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); + $numericTable = ArchiveTableCreator::getNumericTable($date); - $store->markSiteIdsHaveBeenPurged($idSites, $yearMonth); + $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites); + if (count($archiveIds) == 0) { + return; } + + $this->deleteArchiveIds($date, $archiveIds); + + $this->invalidatedReports->markSiteIdsHaveBeenPurged($idSites, $yearMonth); } /** @@ -71,17 +123,14 @@ public function purgeInvalidatedArchives() */ public function purgeOutdatedArchives(Date $dateStart) { - $purgeArchivesOlderThan = self::getOldestTemporaryArchiveToKeepThreshold(); - if (!$purgeArchivesOlderThan) { - return; - } + $purgeArchivesOlderThan = $this->getOldestTemporaryArchiveToKeepThreshold(); $idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan); if (!empty($idArchivesToDelete)) { $this->deleteArchiveIds($dateStart, $idArchivesToDelete); } - $this->deleteArchivesWithPeriodRange($dateStart); + $this->deleteArchivesWithPeriodRange($dateStart); // TODO: this is unrelated to outdated archive purging, should be in its own method Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]", $purgeArchivesOlderThan, @@ -114,13 +163,11 @@ protected function deleteArchivesWithPeriodRange(Date $date) { $numericTable = ArchiveTableCreator::getNumericTable($date); $blobTable = ArchiveTableCreator::getBlobTable($date); - $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days']; - $pastDate = Date::factory('today')->subDay($daysRangesValid)->getDateTime(); - $this->model->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $pastDate); + $this->model->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan); Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]", - $pastDate, $numericTable); + $this->purgeCustomRangesOlderThan, $numericTable); } /** @@ -145,16 +192,52 @@ protected function deleteArchiveIds(Date $date, $idArchivesToDelete) * * @return int|bool Outdated archives older than this timestamp should be purged */ - private static function getOldestTemporaryArchiveToKeepThreshold() + protected function getOldestTemporaryArchiveToKeepThreshold() { $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive(); if (Rules::isBrowserTriggerEnabled()) { // If Browser Archiving is enabled, it is likely there are many more temporary archives // We delete more often which is safe, since reports are re-processed on demand - return Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime(); + return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime(); } // If cron core:archive command is building the reports, we should keep all temporary reports from today - return Date::factory('yesterday')->getDateTime(); + return $this->yesterday->getDateTime(); + } + + private static function getDefaultCustomRangeToPurgeAgeThreshold() + { + $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days']; + return Date::factory('today')->subDay($daysRangesValid)->getDateTime(); + } + + /** + * For tests. + * + * @param Date $yesterday + */ + public function setYesterdayDate(Date $yesterday) + { + $this->yesterday = $yesterday; + } + + /** + * For tests. + * + * @param Date $today + */ + public function setTodayDate(Date $today) + { + $this->today = $today; + } + + /** + * For tests. + * + * @param int $now + */ + public function setNow($now) + { + $this->now = $now; } } \ No newline at end of file diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php new file mode 100644 index 00000000000..bf31d5904b5 --- /dev/null +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -0,0 +1,429 @@ + 1, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-03', + 'date2' => '2015-02-03', + 'period' => 1, + 'ts_archived' => '2015-02-03 12:12:12' + ), + + array( + 'idarchive' => 2, + 'idsite' => 2, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-01', + 'date2' => '2015-02-31', + 'period' => 3, + 'ts_archived' => '2015-02-18 10:10:10' + ), + + array( + 'idarchive' => 3, + 'idsite' => 3, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-04', + 'date2' => '2015-02-10', + 'period' => 2, + 'ts_archived' => '2015-02-10 12:34:56' + ), + + array( + 'idarchive' => 4, + 'idsite' => 1, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-15', + 'date2' => '2015-02-15', + 'period' => 1, + 'ts_archived' => '2015-02-15 08:12:13' + ), + + + // valid temporary + array( // only valid + 'idarchive' => 5, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-27', + 'date2' => '2015-02-27', + 'period' => 1, + 'ts_archived' => '2015-02-27 08:08:08' + ), + + array( + 'idarchive' => 6, + 'idsite' => 2, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-26', + 'date2' => '2015-02-26', + 'period' => 1, + 'ts_archived' => '2015-02-26 07:07:07' + ), + + array( + 'idarchive' => 7, + 'idsite' => 3, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-01', + 'date2' => '2015-02-28', + 'period' => 3, + 'ts_archived' => '2015-02-15 00:00:00' + ), + + // custom ranges + array( + 'idarchive' => 8, + 'idsite' => 1, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-03', + 'date2' => '2015-02-14', + 'period' => 5, + 'ts_archived' => '2015-02-27 00:00:00' + ), + + array( + 'idarchive' => 9, + 'idsite' => 2, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-05', + 'date2' => '2015-02-14', + 'period' => 5, + 'ts_archived' => '2015-02-15 00:00:00' + ), + + array( + 'idarchive' => 10, + 'idsite' => 3, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-05', + 'date2' => '2015-03-05', + 'period' => 5, + 'ts_archived' => '2015-02-26 00:00:00' + ), + + // invalidated + array( + 'idarchive' => 11, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-10', + 'date2' => '2015-02-10', + 'period' => 1, + 'ts_archived' => '2015-02-10 12:13:14' + ), + + array( + 'idarchive' => 12, + 'idsite' => 2, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-08', + 'date2' => '2015-02-14', + 'period' => 2, + 'ts_archived' => '2015-02-15 00:00:00' + ), + + array( + 'idarchive' => 13, + 'idsite' => 3, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-01', + 'date2' => '2015-02-28', + 'period' => 3, + 'ts_archived' => '2015-02-27 13:13:13' + ), + + array( + 'idarchive' => 14, + 'idsite' => 1, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-28', + 'date2' => '2015-02-28', + 'period' => 1, + 'ts_archived' => '2015-02-28 12:12:12' + ), + ); + + /** + * @var Purger + */ + private $archivePurger; + + /** + * @var Date + */ + private $january; + + /** + * @var Date + */ + private $february; + + public function setUp() + { + parent::setUp(); + + $this->january = Date::factory('2015-01-01'); + $this->february = Date::factory('2015-02-01'); + + $this->archivePurger = new Purger(); + $this->archivePurger->setTodayDate(Date::factory('2015-02-27')); + $this->archivePurger->setYesterdayDate(Date::factory('2015-02-26')); + $this->archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp()); + + $this->insertOutdatedArchives($this->january); + $this->insertOutdatedArchives($this->february); + + $this->configureCustomRangePurging(); + } + + public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRangeArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringEnabled() + { + $this->enableBrowserTriggeredArchiving(); + + $this->archivePurger->purgeOutdatedArchives($this->february); + + $this->assertFebruaryTemporaryArchivesPurged($browserTriggeringEnabled = true); + $this->assertFebruaryCustomRangesPurged(); + + $this->assertJanuaryTemporaryArchivesNotPurged(); + } + + public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRangeArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringDisabled() + { + $this->disableBrowserTriggeredArchiving(); + + $this->archivePurger->purgeOutdatedArchives($this->february); + + $this->assertFebruaryTemporaryArchivesPurged($browserTriggeringEnabled = false); + $this->assertFebruaryCustomRangesPurged(); + + $this->assertJanuaryTemporaryArchivesNotPurged(); + } + + public function test_purgeInvalidatedArchives_PurgesCorrectInvalidatedArchives_AndOnlyPurgesDataForDatesAndSites_InInvalidatedReportsDistributedList() + { + $this->setUpInvalidatedReportsDistributedList($dates = array($this->february), $sites = array(1, 3)); + + $this->archivePurger->purgeInvalidatedArchives(); + + // check invalidated archives for idsite = 1, idsite = 3 are purged + $expectedPurgedArchives = array(11, 13); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + + $expectedPresentArchives = array(12, 14); + $this->assertArchivesExist($expectedPresentArchives, $this->february); + + $this->assertJanuaryInvalidatedArchivesNotPurged(); + } + + public function test_purgeInvalidatedArchivesFrom_PurgesCorrectInvalidatedArchives_AndMarksDatesAndSitesAsInvalidated() + { + $this->setUpInvalidatedReportsDistributedList($dates = array($this->february), $sites = array(1, 2, 3)); + + $this->archivePurger->purgeInvalidatedArchivesFrom("2015_01", array(1,2)); + + // check invalidated archives for idsite = 1, idsite = 2 are purged + $expectedPurgedArchives = array(11, 13, 14); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + + $expectedPresentArchives = array(12); + $this->assertArchivesExist($expectedPresentArchives, $this->february); + + $this->assertJanuaryInvalidatedArchivesNotPurged(); + + // assert invalidated reports distributed list has changed + $invalidatedReports = new InvalidatedReports(); + $sitesByYearMonth = $invalidatedReports->getSitesByYearMonthArchiveToPurge(); + + $this->assertEquals(array( + '2015_02' => array(3) + ), $sitesByYearMonth); + } + + private function configureCustomRangePurging() + { + Config::getInstance()->General['purge_date_range_archives_after_X_days'] = 3; + } + + private function enableBrowserTriggeredArchiving() + { + Config::getInstance()->General['enable_browser_archiving_triggering'] = 1; + } + + private function disableBrowserTriggeredArchiving() + { + Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; + } + + /** + * @param Date[] $dates + * @param int[] $sites + */ + private function setUpInvalidatedReportsDistributedList($dates, $sites) + { + $yearMonths = array(); + foreach ($dates as $date) { + $yearMonths[] = $date->toString('Y_m'); + } + + $invalidatedReports = new InvalidatedReports(); + $invalidatedReports->addSitesToPurgeForYearMonths($sites, $yearMonths); + } + + private function insertOutdatedArchives(Date $archiveDate) + { + $dummyArchiveData = $this->getDummyArchiveDataForDate($archiveDate); + + $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); + foreach ($dummyArchiveData as $row) { + // done row + $this->insertTestArchiveRow($numericTable, $row); + + // two metrics + $row['name'] = 'nb_visits'; + $row['value'] = 1; + $this->insertTestArchiveRow($numericTable, $row); + + $row['name'] = 'nb_actions'; + $row['value'] = 2; + $this->insertTestArchiveRow($numericTable, $row); + } + + $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); + foreach ($dummyArchiveData as $row) { + // two blobs + $row['name'] = 'blobname'; + $row['value'] = 'dummyvalue'; + $this->insertTestArchiveRow($blobTable, $row); + + $row['name'] = 'blobname2'; + $row['value'] = 'dummyvalue'; + $this->insertTestArchiveRow($blobTable, $row); + } + } + + private function insertTestArchiveRow($table, $row) + { + $insertSqlTemplate = "INSERT INTO %s (idarchive, idsite, name, value, date1, date2, period, ts_archived) VALUES ('%s')"; + + Db::exec(sprintf($insertSqlTemplate, $table, implode("','", $row))); + } + + private function getDummyArchiveDataForDate($archiveDate) + { + $rows = self::$dummyArchiveData; + foreach ($rows as &$row) { + $row['date1'] = $this->setDateMonthAndYear($row['date1'], $archiveDate); + $row['date2'] = $this->setDateMonthAndYear($row['date1'], $archiveDate); + } + return$rows; + } + + private function setDateMonthAndYear($dateString, Date $archiveDate) + { + return $archiveDate->toString('Y-m') . '-' . Date::factory($dateString)->toString('d'); + } + + private function assertFebruaryTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled) + { + if ($isBrowserTriggeredArchivingEnabled) { + $expectedPurgedArchives = array(1,2,3,4,6,7); + } else { + $expectedPurgedArchives = array(1,2,3,4,5,6,7); + } + + $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + } + + private function assertFebruaryCustomRangesPurged() + { + $expectedPurgedArchives = array(8,9,10); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + } + + private function assertJanuaryTemporaryArchivesNotPurged() + { + $expectedPresentArchives = array(1,2,3,4,5,6,7); + $this->assertArchivesExist($expectedPresentArchives, $this->january); + } + + private function assertJanuaryInvalidatedArchivesNotPurged() + { + $expectedPresentArchives = array(11, 12, 13, 14); + $this->assertArchivesExist($expectedPresentArchives, $this->january); + } + + private function assertArchivesDoNotExist($expectedPurgedArchiveIds, $archiveDate) + { + $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); + $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); + + $numericPurgedArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPurgedArchiveIds); + $this->assertEquals(0, $numericPurgedArchiveCount); + + $blobPurgedArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPurgedArchiveIds); + $this->assertEquals(0, $blobPurgedArchiveCount); + } + + private function assertArchivesExist($expectedPresentArchiveIds, $archiveDate) + { + $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); + $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); + + $numericArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPresentArchiveIds); + $expectedNumericRowCount = count($expectedPresentArchiveIds) * 3; // two metrics + 1 done row + $this->assertEquals($expectedNumericRowCount, $numericArchiveCount); + + $blobArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPresentArchiveIds); + $expectedBlobRowCount = count($expectedPresentArchiveIds) * 2; // two blob rows + $this->assertEquals($expectedBlobRowCount, $blobArchiveCount); + } + + private function getArchiveRowCountWithId($table, $archiveIds) + { + return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE idarchive IN (".implode(',', $archiveIds).")"); + } +} From 5ec61355ba7ca8b59f83efdd1d0573d033b234f4 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Tue, 3 Mar 2015 01:54:35 -0800 Subject: [PATCH 06/31] Get Archive\Purger tests to pass. --- .../Integration/Archive/PurgerTest.php | 75 ++++++++++++++++--- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index bf31d5904b5..63b4969b436 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -184,6 +184,62 @@ class PurgerTest extends IntegrationTestCase 'period' => 1, 'ts_archived' => '2015-02-28 12:12:12' ), + + array( + 'idarchive' => 15, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-27', + 'date2' => '2015-02-27', + 'period' => 1, + 'ts_archived' => '2015-02-28 12:12:12' + ), + + // reprocessed invalidated + array( + 'idarchive' => 16, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-10', + 'date2' => '2015-02-10', + 'period' => 1, + 'ts_archived' => '2015-02-11 12:13:14' + ), + + array( + 'idarchive' => 17, + 'idsite' => 2, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-08', + 'date2' => '2015-02-14', + 'period' => 2, + 'ts_archived' => '2015-02-16 00:00:00' + ), + + array( + 'idarchive' => 18, + 'idsite' => 3, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-01', + 'date2' => '2015-02-28', + 'period' => 3, + 'ts_archived' => '2015-02-28 13:13:13' + ), + + array( + 'idarchive' => 19, + 'idsite' => 1, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-28', + 'date2' => '2015-02-28', + 'period' => 1, + 'ts_archived' => '2015-02-28 16:12:12' // must be late so it doesn't screw up the purgeOutdatedArchives test + ), ); /** @@ -250,10 +306,10 @@ public function test_purgeInvalidatedArchives_PurgesCorrectInvalidatedArchives_A $this->archivePurger->purgeInvalidatedArchives(); // check invalidated archives for idsite = 1, idsite = 3 are purged - $expectedPurgedArchives = array(11, 13); + $expectedPurgedArchives = array(11, 13, 14); $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); - $expectedPresentArchives = array(12, 14); + $expectedPresentArchives = array(12); $this->assertArchivesExist($expectedPresentArchives, $this->february); $this->assertJanuaryInvalidatedArchivesNotPurged(); @@ -263,13 +319,13 @@ public function test_purgeInvalidatedArchivesFrom_PurgesCorrectInvalidatedArchiv { $this->setUpInvalidatedReportsDistributedList($dates = array($this->february), $sites = array(1, 2, 3)); - $this->archivePurger->purgeInvalidatedArchivesFrom("2015_01", array(1,2)); + $this->archivePurger->purgeInvalidatedArchivesFrom("2015_02", array(1,2)); // check invalidated archives for idsite = 1, idsite = 2 are purged - $expectedPurgedArchives = array(11, 13, 14); + $expectedPurgedArchives = array(11, 12, 14); $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); - $expectedPresentArchives = array(12); + $expectedPresentArchives = array(13); $this->assertArchivesExist($expectedPresentArchives, $this->february); $this->assertJanuaryInvalidatedArchivesNotPurged(); @@ -278,9 +334,8 @@ public function test_purgeInvalidatedArchivesFrom_PurgesCorrectInvalidatedArchiv $invalidatedReports = new InvalidatedReports(); $sitesByYearMonth = $invalidatedReports->getSitesByYearMonthArchiveToPurge(); - $this->assertEquals(array( - '2015_02' => array(3) - ), $sitesByYearMonth); + $this->assertArrayHasKey('2015_02', $sitesByYearMonth); + $this->assertEquals(array(3), array_values($sitesByYearMonth['2015_02'])); } private function configureCustomRangePurging() @@ -370,9 +425,9 @@ private function setDateMonthAndYear($dateString, Date $archiveDate) private function assertFebruaryTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled) { if ($isBrowserTriggeredArchivingEnabled) { - $expectedPurgedArchives = array(1,2,3,4,6,7); + $expectedPurgedArchives = array(1,2,3,4,6,7); // only archives from 2 hours before "now" are purged } else { - $expectedPurgedArchives = array(1,2,3,4,5,6,7); + $expectedPurgedArchives = array(1,2,3,4,7); // only archives before start of "yesterday" are purged } $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); From bc71485e2fe243d0a6c54192273df888c3f7c1f0 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Tue, 3 Mar 2015 17:16:53 -0800 Subject: [PATCH 07/31] Add command to force archive purging. --- .../Commands/PurgeOldArchiveData.php | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php new file mode 100644 index 00000000000..9548efb6e07 --- /dev/null +++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php @@ -0,0 +1,123 @@ +archivePurger = $archivePurger ?: new Purger(); + } + + protected function configure() + { + $this->setName('core:purge-old-archive-data'); + $this->setDescription('Purges old and invalid archive data from archive tables.'); + $this->addArgument("dates", InputArgument::IS_ARRAY | InputArgument::OPTIONAL, + "The months of the archive tables to purge data from. By default, only deletes from the current month. Use '" . self::ALL_DATES_STRING. "' for all dates.", + array(Date::today()->toString())); + $this->addOption('exclude-outdated', null, InputOption::VALUE_NONE, "Do not purge outdated archive data."); + $this->addOption('exclude-invalidated', null, InputOption::VALUE_NONE, "Do not purge invalidated archive data."); + $this->setHelp("By default old and invalidated archives are purged. Custom ranges are also purged with outdated archives.\n\n" + . "Note: archive purging is done during scheduled task execution, so under normal circumstances, you should not need to " + . "run this command manually."); + + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $self = $this; + + $dates = $this->getDatesToPurgeFor($input); + + $excludeOutdated = $input->getOption('exclude-outdated'); + if ($excludeOutdated) { + $output->writeln("Skipping purge outdated archive data."); + } else { + foreach ($dates as $date) { + $message = sprintf("Purging outdated archives for %s...", $date->toString('Y_m')); + $this->performTimedPurging($output, $message, function () use ($date, $self) { + $self->archivePurger->purgeOutdatedArchives($date); + }); + } + } + + $excludeInvalidated = $input->getOption('exclude-invalidated'); + if ($excludeInvalidated) { + $output->writeln("Skipping purge invalidated archive data."); + } else { + $this->performTimedPurging($output, "Purging invalidated archives...", function () use ($self) { + $self->archivePurger->purgeInvalidatedArchives(); + }); + } + } + + /** + * @param InputInterface $input + * @return Date[] + */ + private function getDatesToPurgeFor(InputInterface $input) + { + $dates = array(); + + $dateSpecifier = $input->getArgument('dates'); + if (count($dateSpecifier) === 1 + && reset($dateSpecifier) == self::ALL_DATES_STRING + ) { + foreach (ArchiveTableCreator::getTablesArchivesInstalled() as $table) { + $tableDate = ArchiveTableCreator::getDateFromTableName($table); + + list($year, $month) = explode('_', $tableDate); + + $dates[] = Date::factory($year . '-' . $month . '-' . '01'); + } + } else { + foreach ($dateSpecifier as $date) { + $dates[] = Date::factory($date); + } + } + + return $dates; + } + + private function performTimedPurging(OutputInterface $output, $startMessage, $callback) + { + $timer = new Timer(); + + $output->write($startMessage); + + $callback(); + + $output->writeln("Done. [" . $timer->__toString() . "]"); + } +} From 35a6c4c8b051d67d7b23c20c12e8ff77be3a06ca Mon Sep 17 00:00:00 2001 From: diosmosis Date: Wed, 4 Mar 2015 02:18:38 -0800 Subject: [PATCH 08/31] Refs #7181, decouple invalidated archive purging from option table & site entity, no longer need to depend on option existing in order to successfully purge invalidated archives. --- core/Archive/Purger.php | 16 ++++++++-------- core/DataAccess/Model.php | 18 ++++++++++++++++++ .../PHPUnit/Integration/Archive/PurgerTest.php | 18 ++++++------------ 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index 0b85ff2a068..d4e55993a9c 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -86,25 +86,25 @@ public function __construct(Model $model = null, Date $purgeCustomRangesOlderTha */ public function purgeInvalidatedArchives() { - // TODO: why does it only look in specified sites to purge instead of all sites? $idSitesByYearMonth = $this->invalidatedReports->getSitesByYearMonthArchiveToPurge(); - foreach ($idSitesByYearMonth as $yearMonth => $idSites) { - $this->purgeInvalidatedArchivesFrom($yearMonth, $idSites); + foreach ($idSitesByYearMonth as $yearMonth => $idSites) { // TODO: change the option to store $yearMonths as values? perhaps not necessary right now + $this->purgeInvalidatedArchivesFrom($yearMonth); } } /** * TODO */ - public function purgeInvalidatedArchivesFrom($yearMonth, $idSites) + public function purgeInvalidatedArchivesFrom($yearMonth) { - if (empty($idSites)) { - return; - } - $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); $numericTable = ArchiveTableCreator::getNumericTable($date); + // we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows, + // so we first look for sites w/ invalidated archives, and use this as a constraint in getInvalidatedArchiveIdsSafeToDelete() below. + // the constraint will hit an INDEX and speed things up. + $idSites = $this->model->getSitesWithInvalidatedArchive($numericTable); + $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites); if (count($archiveIds) == 0) { return; diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 0e4b2845a7c..9653e0ec58f 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -237,6 +237,24 @@ public function insertRecord($tableName, $fields, $record, $name, $value) return true; } + /** + * TODO + * TODO: tests? there doesn't seem to be tests for other methods here. + * + * @param string $numericTable + * @return int[] + */ + public function getSitesWithInvalidatedArchive($numericTable) + { + $rows = Db::fetchAll("SELECT DISTINCT idsite FROM `$numericTable` WHERE name LIKE 'done%' AND value = " . ArchiveWriter::DONE_INVALIDATED); + + $result = array(); + foreach ($rows as $row) { + $result[] = $row['idsite']; + } + return $result; + } + /** * Returns the SQL condition used to find successfully completed archives that * this instance is querying for. diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index 63b4969b436..a066c082e65 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -306,36 +306,30 @@ public function test_purgeInvalidatedArchives_PurgesCorrectInvalidatedArchives_A $this->archivePurger->purgeInvalidatedArchives(); // check invalidated archives for idsite = 1, idsite = 3 are purged - $expectedPurgedArchives = array(11, 13, 14); + $expectedPurgedArchives = array(11, 12, 13, 14); $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); - $expectedPresentArchives = array(12); - $this->assertArchivesExist($expectedPresentArchives, $this->february); - $this->assertJanuaryInvalidatedArchivesNotPurged(); } - public function test_purgeInvalidatedArchivesFrom_PurgesCorrectInvalidatedArchives_AndMarksDatesAndSitesAsInvalidated() + public function test_purgeInvalidatedArchivesFrom_PurgesAllInvalidatedArchives_AndMarksDatesAndSitesAsInvalidated() { - $this->setUpInvalidatedReportsDistributedList($dates = array($this->february), $sites = array(1, 2, 3)); + $this->setUpInvalidatedReportsDistributedList($dates = array($this->january, $this->february), $sites = array(1, 2, 3)); $this->archivePurger->purgeInvalidatedArchivesFrom("2015_02", array(1,2)); // check invalidated archives for idsite = 1, idsite = 2 are purged - $expectedPurgedArchives = array(11, 12, 14); + $expectedPurgedArchives = array(11, 12, 13, 14); $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); - $expectedPresentArchives = array(13); - $this->assertArchivesExist($expectedPresentArchives, $this->february); - $this->assertJanuaryInvalidatedArchivesNotPurged(); // assert invalidated reports distributed list has changed $invalidatedReports = new InvalidatedReports(); $sitesByYearMonth = $invalidatedReports->getSitesByYearMonthArchiveToPurge(); - $this->assertArrayHasKey('2015_02', $sitesByYearMonth); - $this->assertEquals(array(3), array_values($sitesByYearMonth['2015_02'])); + $this->assertArrayNotHasKey('2015_02', $sitesByYearMonth); + $this->assertArrayHasKey('2015_01', $sitesByYearMonth); } private function configureCustomRangePurging() From b92d60f9c30ef2c7ed06f43ac1dd5f29917a55ab Mon Sep 17 00:00:00 2001 From: diosmosis Date: Wed, 4 Mar 2015 18:07:19 -0800 Subject: [PATCH 09/31] Refs #7181, optimize tables at end of core:purge-old-archive-data command, so table size will be reduced and not just row count. --- .../Commands/PurgeOldArchiveData.php | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php index 9548efb6e07..2f2c465b8b6 100644 --- a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php +++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php @@ -22,6 +22,7 @@ /** * TODO + * TODO: command tests */ class PurgeOldArchiveData extends ConsoleCommand { @@ -48,6 +49,7 @@ protected function configure() array(Date::today()->toString())); $this->addOption('exclude-outdated', null, InputOption::VALUE_NONE, "Do not purge outdated archive data."); $this->addOption('exclude-invalidated', null, InputOption::VALUE_NONE, "Do not purge invalidated archive data."); + $this->addOption('skip-optimize-tables', null, InputOption::VALUE_NONE, "Do not run OPTIMIZE TABLES query on affected archive tables."); $this->setHelp("By default old and invalidated archives are purged. Custom ranges are also purged with outdated archives.\n\n" . "Note: archive purging is done during scheduled task execution, so under normal circumstances, you should not need to " . "run this command manually."); @@ -80,6 +82,13 @@ protected function execute(InputInterface $input, OutputInterface $output) $self->archivePurger->purgeInvalidatedArchives(); }); } + + $skipOptimizeTables = $input->getOption('skip-optimize-tables'); + if ($skipOptimizeTables) { + $output->writeln("Skipping OPTIMIZE TABLES."); + } else { + $this->optimizeArchiveTables($output, $dates); + } } /** @@ -120,4 +129,24 @@ private function performTimedPurging(OutputInterface $output, $startMessage, $ca $output->writeln("Done. [" . $timer->__toString() . "]"); } + + /** + * @param Date[] $dates + */ + private function optimizeArchiveTables(OutputInterface $output, $dates) + { + $output->writeln("Optimizing archive tables..."); + + foreach ($dates as $date) { + $numericTable = ArchiveTableCreator::getNumericTable($date); + $this->performTimedPurging($output, "Optimizing table $numericTable...", function () use ($numericTable) { + Db::optimizeTables($numericTable); + }); + + $blobTable = ArchiveTableCreator::getBlobTable($date); + $this->performTimedPurging($output, "Optimizing table $blobTable...", function () use ($blobTable) { + Db::optimizeTables($blobTable); + }); + } + } } From 5b96a6ecfe6bf42195b400daaca5331134f6ca87 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Wed, 4 Mar 2015 18:25:18 -0800 Subject: [PATCH 10/31] Light refactoring to Archive\Purger class. --- core/Archive/Purger.php | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index d4e55993a9c..34d0bc34be6 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -34,13 +34,6 @@ class Purger */ private $model; - /** - * TODO: what does this class even do? - * - * @var InvalidatedReports - */ - private $invalidatedReports; - /** * TODO * @@ -72,7 +65,6 @@ class Purger public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null) { $this->model = $model ?: new Model(); - $this->invalidatedReports = new InvalidatedReports(); $this->purgeCustomRangesOlderThan = $purgeCustomRangesOlderThan ?: self::getDefaultCustomRangeToPurgeAgeThreshold(); @@ -86,9 +78,13 @@ public function __construct(Model $model = null, Date $purgeCustomRangesOlderTha */ public function purgeInvalidatedArchives() { - $idSitesByYearMonth = $this->invalidatedReports->getSitesByYearMonthArchiveToPurge(); + $invalidatedReports = new InvalidatedReports(); + + $idSitesByYearMonth = $invalidatedReports->getSitesByYearMonthArchiveToPurge(); foreach ($idSitesByYearMonth as $yearMonth => $idSites) { // TODO: change the option to store $yearMonths as values? perhaps not necessary right now $this->purgeInvalidatedArchivesFrom($yearMonth); + + $invalidatedReports->markSiteIdsHaveBeenPurged($idSites, $yearMonth); } } @@ -102,17 +98,18 @@ public function purgeInvalidatedArchivesFrom($yearMonth) // we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows, // so we first look for sites w/ invalidated archives, and use this as a constraint in getInvalidatedArchiveIdsSafeToDelete() below. - // the constraint will hit an INDEX and speed things up. + // the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete(). $idSites = $this->model->getSitesWithInvalidatedArchive($numericTable); + if (empty($idSites)) { + return; + } $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites); - if (count($archiveIds) == 0) { + if (empty($archiveIds)) { return; } $this->deleteArchiveIds($date, $archiveIds); - - $this->invalidatedReports->markSiteIdsHaveBeenPurged($idSites, $yearMonth); } /** From 13bdf6749222c7335e81511480ac867b3a859cc3 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Wed, 4 Mar 2015 19:08:19 -0800 Subject: [PATCH 11/31] Refs #7181, remove site IDs from one of InvalidatedReports' distributed lists, fix new purging command for php 5.3, and update relevant methods to not use site IDs when purging/invalidating. --- core/Archive/Purger.php | 17 ++++-- core/DataAccess/ArchiveInvalidator.php | 2 +- core/DataAccess/InvalidatedReports.php | 55 +++++-------------- .../Commands/PurgeOldArchiveData.php | 15 +++-- .../Integration/Archive/PurgerTest.php | 44 ++++++++------- 5 files changed, 58 insertions(+), 75 deletions(-) diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index 34d0bc34be6..dc6b8316cf9 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -80,20 +80,25 @@ public function purgeInvalidatedArchives() { $invalidatedReports = new InvalidatedReports(); - $idSitesByYearMonth = $invalidatedReports->getSitesByYearMonthArchiveToPurge(); - foreach ($idSitesByYearMonth as $yearMonth => $idSites) { // TODO: change the option to store $yearMonths as values? perhaps not necessary right now - $this->purgeInvalidatedArchivesFrom($yearMonth); + $idSitesByYearMonth = $invalidatedReports->getYearMonthArchivesToPurge(); + foreach ($idSitesByYearMonth as $yearMonth) { + try { + $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); + } catch (\Exception $ex) { + continue; // invalid year month in distributed list + } + + $this->purgeInvalidatedArchivesFrom($date); - $invalidatedReports->markSiteIdsHaveBeenPurged($idSites, $yearMonth); + $invalidatedReports->markArchiveTablePurged($yearMonth); } } /** * TODO */ - public function purgeInvalidatedArchivesFrom($yearMonth) + public function purgeInvalidatedArchivesFrom(Date $date) { - $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); $numericTable = ArchiveTableCreator::getNumericTable($date); // we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows, diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php index cc0d7d0941d..30a084d97c9 100644 --- a/core/DataAccess/ArchiveInvalidator.php +++ b/core/DataAccess/ArchiveInvalidator.php @@ -319,7 +319,7 @@ private function persistInvalidatedArchives(array $idSites, $datesByMonth) $store = new InvalidatedReports(); $store->addInvalidatedSitesToReprocess($idSites); - $store->addSitesToPurgeForYearMonths($idSites, $yearMonths); + $store->addArchiveTablesToPurge($yearMonths); } private static function getModel() diff --git a/core/DataAccess/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php index 64f863e0adb..c940c7ea3b0 100644 --- a/core/DataAccess/InvalidatedReports.php +++ b/core/DataAccess/InvalidatedReports.php @@ -29,52 +29,36 @@ class InvalidatedReports /** * Mark the sites IDs and Dates as being invalidated, so we can purge them later on. * - * @param array $idSites * @param array $yearMonths */ - public function addSitesToPurgeForYearMonths(array $idSites, $yearMonths) + public function addArchiveTablesToPurge($yearMonths) { - $idSitesByYearMonth = $this->getSitesByYearMonthToPurge(); + $yearMonthsToPurge = $this->getYearMonthArchivesToPurge(); + $yearMonthsToPurge = array_merge($yearMonthsToPurge, $yearMonths); + $yearMonthsToPurge = array_unique($yearMonthsToPurge); - foreach($yearMonths as $yearMonthToPurge) { - - if(isset($idSitesByYearMonth[$yearMonthToPurge])) { - $existingIdSitesToPurge = $idSitesByYearMonth[$yearMonthToPurge]; - $idSites = array_merge($existingIdSitesToPurge, $idSites); - $idSites = array_unique($idSites); - } - $idSitesByYearMonth[$yearMonthToPurge] = $idSites; - } - $this->persistSitesByYearMonthToPurge($idSitesByYearMonth); + $this->persistYearMonthArchivesToPurge($yearMonthsToPurge); } /** * Returns the list of websites IDs for which invalidated archives can be purged. */ - public function getSitesByYearMonthArchiveToPurge() + public function getYearMonthArchivesToPurge() { - $idSitesByYearMonth = $this->getSitesByYearMonthToPurge(); - - // From this list we remove the websites that are not yet re-processed - // so we don't purge them before they were re-processed - $idSitesNotYetReprocessed = $this->getSitesToReprocess(); - - foreach($idSitesByYearMonth as $yearMonth => &$idSites) { - $idSites = array_diff($idSites, $idSitesNotYetReprocessed); - } - return $idSitesByYearMonth; + return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE); } - public function markSiteIdsHaveBeenPurged(array $idSites, $yearMonth) + public function markArchiveTablePurged($yearMonth) { - $idSitesByYearMonth = $this->getSitesByYearMonthToPurge(); + $yearMonthsToPurge = $this->getYearMonthArchivesToPurge(); - if(!isset($idSitesByYearMonth[$yearMonth])) { + $existingIndex = array_search($yearMonth, $yearMonthsToPurge); + if ($existingIndex === false) { return; } - $idSitesByYearMonth[$yearMonth] = array_diff($idSitesByYearMonth[$yearMonth], $idSites); - $this->persistSitesByYearMonthToPurge($idSitesByYearMonth); + unset($yearMonthsToPurge[$existingIndex]); + $this->persistYearMonthArchivesToPurge($yearMonthsToPurge); } /** @@ -116,14 +100,6 @@ public function getSitesToReprocess() return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS); } - /** - * @return array|false|mixed|string - */ - private function getSitesByYearMonthToPurge() - { - return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE); - } - /** * @param $websiteIdsInvalidated */ @@ -155,14 +131,11 @@ private function getArrayValueFromOptionName($optionName) /** * @param $idSitesByYearMonth */ - private function persistSitesByYearMonthToPurge($idSitesByYearMonth) + private function persistYearMonthArchivesToPurge($idSitesByYearMonth) { // remove dates for which there are no sites to purge $idSitesByYearMonth = array_filter($idSitesByYearMonth); Option::set(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE, serialize($idSitesByYearMonth)); } - - - } \ No newline at end of file diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php index 2f2c465b8b6..23884736a0f 100644 --- a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php +++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php @@ -58,7 +58,7 @@ protected function configure() protected function execute(InputInterface $input, OutputInterface $output) { - $self = $this; + $archivePurger = $this->archivePurger; $dates = $this->getDatesToPurgeFor($input); @@ -68,8 +68,8 @@ protected function execute(InputInterface $input, OutputInterface $output) } else { foreach ($dates as $date) { $message = sprintf("Purging outdated archives for %s...", $date->toString('Y_m')); - $this->performTimedPurging($output, $message, function () use ($date, $self) { - $self->archivePurger->purgeOutdatedArchives($date); + $this->performTimedPurging($output, $message, function () use ($date, $archivePurger) { + $archivePurger->purgeOutdatedArchives($date); }); } } @@ -78,9 +78,12 @@ protected function execute(InputInterface $input, OutputInterface $output) if ($excludeInvalidated) { $output->writeln("Skipping purge invalidated archive data."); } else { - $this->performTimedPurging($output, "Purging invalidated archives...", function () use ($self) { - $self->archivePurger->purgeInvalidatedArchives(); - }); + foreach ($dates as $date) { + $message = sprintf("Purging invalidated archives for %s...", $date->toString('Y_m')); + $this->performTimedPurging($output, $message, function () use ($archivePurger, $date) { + $archivePurger->purgeInvalidatedArchivesFrom($date); + }); + } } $skipOptimizeTables = $input->getOption('skip-optimize-tables'); diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index a066c082e65..e75b43c3432 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -301,35 +301,26 @@ public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRan public function test_purgeInvalidatedArchives_PurgesCorrectInvalidatedArchives_AndOnlyPurgesDataForDatesAndSites_InInvalidatedReportsDistributedList() { - $this->setUpInvalidatedReportsDistributedList($dates = array($this->february), $sites = array(1, 3)); + $this->setUpInvalidatedReportsDistributedList($dates = array($this->february)); $this->archivePurger->purgeInvalidatedArchives(); - // check invalidated archives for idsite = 1, idsite = 3 are purged - $expectedPurgedArchives = array(11, 12, 13, 14); - $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); - + $this->assertFebruaryInvalidatedArchivesPurged(); $this->assertJanuaryInvalidatedArchivesNotPurged(); + + // assert invalidated reports distributed list has changed + $invalidatedReports = new InvalidatedReports(); + $yearMonths = $invalidatedReports->getYearMonthArchivesToPurge(); + + $this->assertEmpty($yearMonths); } public function test_purgeInvalidatedArchivesFrom_PurgesAllInvalidatedArchives_AndMarksDatesAndSitesAsInvalidated() { - $this->setUpInvalidatedReportsDistributedList($dates = array($this->january, $this->february), $sites = array(1, 2, 3)); - - $this->archivePurger->purgeInvalidatedArchivesFrom("2015_02", array(1,2)); - - // check invalidated archives for idsite = 1, idsite = 2 are purged - $expectedPurgedArchives = array(11, 12, 13, 14); - $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + $this->archivePurger->purgeInvalidatedArchivesFrom($this->february); + $this->assertFebruaryInvalidatedArchivesPurged(); $this->assertJanuaryInvalidatedArchivesNotPurged(); - - // assert invalidated reports distributed list has changed - $invalidatedReports = new InvalidatedReports(); - $sitesByYearMonth = $invalidatedReports->getSitesByYearMonthArchiveToPurge(); - - $this->assertArrayNotHasKey('2015_02', $sitesByYearMonth); - $this->assertArrayHasKey('2015_01', $sitesByYearMonth); } private function configureCustomRangePurging() @@ -351,7 +342,7 @@ private function disableBrowserTriggeredArchiving() * @param Date[] $dates * @param int[] $sites */ - private function setUpInvalidatedReportsDistributedList($dates, $sites) + private function setUpInvalidatedReportsDistributedList($dates) { $yearMonths = array(); foreach ($dates as $date) { @@ -359,7 +350,7 @@ private function setUpInvalidatedReportsDistributedList($dates, $sites) } $invalidatedReports = new InvalidatedReports(); - $invalidatedReports->addSitesToPurgeForYearMonths($sites, $yearMonths); + $invalidatedReports->addArchiveTablesToPurge($yearMonths); } private function insertOutdatedArchives(Date $archiveDate) @@ -475,4 +466,15 @@ private function getArchiveRowCountWithId($table, $archiveIds) { return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE idarchive IN (".implode(',', $archiveIds).")"); } + + private function assertFebruaryInvalidatedArchivesPurged() + { + // check invalidated archives for all sites are purged + $expectedPurgedArchives = array(11, 12, 13, 14); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + + // check archive 15 is not purged since it doesn't have newer DONE_OK/DONE_TEMPORARY archive + $expectedExistingArchives = array(15); + $this->assertArchivesExist($expectedExistingArchives, $this->february); + } } From 67677a192e220833ca0fc0e3279e3f792f1f1af5 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Wed, 4 Mar 2015 20:04:51 -0800 Subject: [PATCH 12/31] Refs #7181, move distributed list used to hold archives to purge to separate class from InvalidatedReports. Since list is only used to purge during scheduled tasks, moved to CoreAdminHome plugin where related task resides. Introduced common base type core\Concurrency\DistributedList in order to avoid introducing code redundancy when splitting InvalidatedReports up. --- core/Archive/Purger.php | 9 +- core/Concurrency/DistributedList.php | 109 ++++++++++++++++++ core/DataAccess/ArchiveInvalidator.php | 5 +- core/DataAccess/InvalidatedReports.php | 47 -------- .../Tasks/ArchivesToPurgeDistributedList.php | 38 ++++++ .../Integration/Archive/PurgerTest.php | 10 +- 6 files changed, 161 insertions(+), 57 deletions(-) create mode 100644 core/Concurrency/DistributedList.php create mode 100644 plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index dc6b8316cf9..ac5e91a2bf5 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -17,6 +17,7 @@ use Piwik\Db; use Piwik\Log; use Piwik\Piwik; +use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; /** * @@ -78,10 +79,10 @@ public function __construct(Model $model = null, Date $purgeCustomRangesOlderTha */ public function purgeInvalidatedArchives() { - $invalidatedReports = new InvalidatedReports(); + $archivesToPurge = new ArchivesToPurgeDistributedList(); - $idSitesByYearMonth = $invalidatedReports->getYearMonthArchivesToPurge(); - foreach ($idSitesByYearMonth as $yearMonth) { + $yearMonths = $archivesToPurge->getAll(); + foreach ($yearMonths as $yearMonth) { try { $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); } catch (\Exception $ex) { @@ -90,7 +91,7 @@ public function purgeInvalidatedArchives() $this->purgeInvalidatedArchivesFrom($date); - $invalidatedReports->markArchiveTablePurged($yearMonth); + $archivesToPurge->remove($yearMonth); } } diff --git a/core/Concurrency/DistributedList.php b/core/Concurrency/DistributedList.php new file mode 100644 index 00000000000..2f37fe0e4bf --- /dev/null +++ b/core/Concurrency/DistributedList.php @@ -0,0 +1,109 @@ +optionName = $optionName; + } + + /** + * TODO + * + * @return array + */ + public function getAll() + { + Option::clearCachedOption($this->optionName); + $array = Option::get($this->optionName); + + if ($array + && ($array = unserialize($array)) + && count($array) + ) { + return $array; + } + return array(); + } + + /** + * TODO + * + * @param string[] $items + */ + public function setAll($items) + { + foreach ($items as &$item) { + $item = (string)$item; + } + + Option::set($this->optionName, serialize($items)); + } + + /** + * TODO + * + * @param string|array $item + */ + public function add($item) + { + $allItems = $this->getAll(); + if (is_array($item)) { + $allItems = array_merge($allItems, $item); + } else { + $allItems[] = $item; + } + + $this->setAll($allItems); + } + + /** + * TODO + * TODO: support removing multiple + * + * @param string|array $items + */ + public function remove($items) + { + if (!is_array($items)) { + $items = array($items); + } + + $allItems = $this->getAll(); + + foreach ($items as $item) { + $existingIndex = array_search($item, $allItems); + if ($existingIndex === false) { + return; + } + + unset($allItems[$existingIndex]); + } + + $this->setAll($allItems); + } +} \ No newline at end of file diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php index 30a084d97c9..deffefa622a 100644 --- a/core/DataAccess/ArchiveInvalidator.php +++ b/core/DataAccess/ArchiveInvalidator.php @@ -12,6 +12,7 @@ use Piwik\Date; use Piwik\Db; use Piwik\Option; +use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; use Piwik\Plugins\PrivacyManager\PrivacyManager; use Piwik\Period; use Piwik\Period\Week; @@ -319,7 +320,9 @@ private function persistInvalidatedArchives(array $idSites, $datesByMonth) $store = new InvalidatedReports(); $store->addInvalidatedSitesToReprocess($idSites); - $store->addArchiveTablesToPurge($yearMonths); + + $archivesToPurge = new ArchivesToPurgeDistributedList(); + $archivesToPurge->add($yearMonths); } private static function getModel() diff --git a/core/DataAccess/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php index c940c7ea3b0..3dcc7642f3c 100644 --- a/core/DataAccess/InvalidatedReports.php +++ b/core/DataAccess/InvalidatedReports.php @@ -24,42 +24,6 @@ class InvalidatedReports { const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds'; - const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds'; - - /** - * Mark the sites IDs and Dates as being invalidated, so we can purge them later on. - * - * @param array $yearMonths - */ - public function addArchiveTablesToPurge($yearMonths) - { - $yearMonthsToPurge = $this->getYearMonthArchivesToPurge(); - $yearMonthsToPurge = array_merge($yearMonthsToPurge, $yearMonths); - $yearMonthsToPurge = array_unique($yearMonthsToPurge); - - $this->persistYearMonthArchivesToPurge($yearMonthsToPurge); - } - - /** - * Returns the list of websites IDs for which invalidated archives can be purged. - */ - public function getYearMonthArchivesToPurge() - { - return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE); - } - - public function markArchiveTablePurged($yearMonth) - { - $yearMonthsToPurge = $this->getYearMonthArchivesToPurge(); - - $existingIndex = array_search($yearMonth, $yearMonthsToPurge); - if ($existingIndex === false) { - return; - } - - unset($yearMonthsToPurge[$existingIndex]); - $this->persistYearMonthArchivesToPurge($yearMonthsToPurge); - } /** * Record those website IDs as having been invalidated @@ -127,15 +91,4 @@ private function getArrayValueFromOptionName($optionName) } return array(); } - - /** - * @param $idSitesByYearMonth - */ - private function persistYearMonthArchivesToPurge($idSitesByYearMonth) - { - // remove dates for which there are no sites to purge - $idSitesByYearMonth = array_filter($idSitesByYearMonth); - - Option::set(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE, serialize($idSitesByYearMonth)); - } } \ No newline at end of file diff --git a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php new file mode 100644 index 00000000000..29668098d6e --- /dev/null +++ b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php @@ -0,0 +1,38 @@ +assertJanuaryInvalidatedArchivesNotPurged(); // assert invalidated reports distributed list has changed - $invalidatedReports = new InvalidatedReports(); - $yearMonths = $invalidatedReports->getYearMonthArchivesToPurge(); + $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList(); + $yearMonths = $archivesToPurgeDistributedList->getAll(); $this->assertEmpty($yearMonths); } @@ -340,7 +341,6 @@ private function disableBrowserTriggeredArchiving() /** * @param Date[] $dates - * @param int[] $sites */ private function setUpInvalidatedReportsDistributedList($dates) { @@ -349,8 +349,8 @@ private function setUpInvalidatedReportsDistributedList($dates) $yearMonths[] = $date->toString('Y_m'); } - $invalidatedReports = new InvalidatedReports(); - $invalidatedReports->addArchiveTablesToPurge($yearMonths); + $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList(); + $archivesToPurgeDistributedList->add($yearMonths); } private function insertOutdatedArchives(Date $archiveDate) From 534798ae6b20b6e5f8660a372a97162b8a0e90eb Mon Sep 17 00:00:00 2001 From: diosmosis Date: Wed, 4 Mar 2015 22:19:53 -0800 Subject: [PATCH 13/31] Refs #7181, rename remaining InvalidatedReports class to SitesToReprocessDistributedList and move to Piwik\CronArchive since CronArchive is only consumer of the list. Made the list derive from Piwik\Concurrency\DistributedList. --- core/Archive/Purger.php | 2 +- core/CronArchive.php | 18 ++-- .../SitesToReprocessDistributedList.php | 46 +++++++++ core/DataAccess/ArchiveInvalidator.php | 9 +- core/DataAccess/InvalidatedReports.php | 94 ------------------- .../Tasks/ArchivesToPurgeDistributedList.php | 1 - .../Integration/Archive/PurgerTest.php | 2 +- ...omVariablesSegmentMatchVisitorTypeTest.php | 2 +- 8 files changed, 63 insertions(+), 111 deletions(-) create mode 100644 core/CronArchive/SitesToReprocessDistributedList.php delete mode 100644 core/DataAccess/InvalidatedReports.php diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index ac5e91a2bf5..6f34fddeada 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -11,7 +11,7 @@ use Piwik\ArchiveProcessor\Rules; use Piwik\Config; use Piwik\DataAccess\ArchiveTableCreator; -use Piwik\DataAccess\InvalidatedReports; +use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\DataAccess\Model; use Piwik\Date; use Piwik\Db; diff --git a/core/CronArchive.php b/core/CronArchive.php index 5c1b497649a..9e592e58d2f 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -16,7 +16,7 @@ use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Metrics\Formatter; use Piwik\Period\Factory as PeriodFactory; -use Piwik\DataAccess\InvalidatedReports; +use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI; use Piwik\Plugins\SitesManager\API as APISitesManager; use Piwik\Plugins\UsersManager\API as APIUsersManager; @@ -542,8 +542,8 @@ private function archiveSingleSite($idSite, $requestsBefore) if(!$success) { // cancel marking the site as reprocessed if($websiteInvalidatedShouldReprocess) { - $store = new InvalidatedReports(); - $store->addInvalidatedSitesToReprocess(array($idSite)); + $store = new SitesToReprocessDistributedList(); + $store->add($idSite); } } @@ -657,8 +657,8 @@ protected function processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay // does not archive the same idSite $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite); if ($websiteInvalidatedShouldReprocess) { - $store = new InvalidatedReports(); - $store->storeSiteIsReprocessed($idSite); + $store = new SitesToReprocessDistributedList(); + $store->remove($idSite); } // when some data was purged from this website @@ -685,8 +685,8 @@ protected function processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay // cancel marking the site as reprocessed if($websiteInvalidatedShouldReprocess) { - $store = new InvalidatedReports(); - $store->addInvalidatedSitesToReprocess(array($idSite)); + $store = new SitesToReprocessDistributedList(); + $store->add($idSite); } $this->logError("Empty or invalid response '$content' for website id $idSite, " . $timerWebsite->__toString() . ", skipping"); @@ -1087,8 +1087,8 @@ protected function initPiwikHost($piwikUrl = false) private function updateIdSitesInvalidatedOldReports() { - $store = new InvalidatedReports(); - $this->idSitesInvalidatedOldReports = $store->getSitesToReprocess(); + $store = new SitesToReprocessDistributedList(); + $this->idSitesInvalidatedOldReports = $store->getAll(); } /** diff --git a/core/CronArchive/SitesToReprocessDistributedList.php b/core/CronArchive/SitesToReprocessDistributedList.php new file mode 100644 index 00000000000..c16062105ee --- /dev/null +++ b/core/CronArchive/SitesToReprocessDistributedList.php @@ -0,0 +1,46 @@ +addInvalidatedSitesToReprocess($idSites); + $store = new SitesToReprocessDistributedList(); + $store->add($idSites); $archivesToPurge = new ArchivesToPurgeDistributedList(); $archivesToPurge->add($yearMonths); diff --git a/core/DataAccess/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php deleted file mode 100644 index 3dcc7642f3c..00000000000 --- a/core/DataAccess/InvalidatedReports.php +++ /dev/null @@ -1,94 +0,0 @@ -getSitesToReprocess(); - $siteIdsToReprocess = array_merge($siteIdsToReprocess, $idSites); - $this->setSitesToReprocess($siteIdsToReprocess); - } - - - /** - * @param $idSite - */ - public function storeSiteIsReprocessed($idSite) - { - $siteIdsToReprocess = $this->getSitesToReprocess(); - - if (count($siteIdsToReprocess)) { - $found = array_search($idSite, $siteIdsToReprocess); - if ($found !== false) { - unset($siteIdsToReprocess[$found]); - $this->setSitesToReprocess($siteIdsToReprocess); - } - } - } - - /** - * Returns array of idSites to force re-process next time core:archive command runs - * - * @return array of id sites - */ - public function getSitesToReprocess() - { - return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS); - } - - /** - * @param $websiteIdsInvalidated - */ - private function setSitesToReprocess($websiteIdsInvalidated) - { - $websiteIdsInvalidated = array_unique($websiteIdsInvalidated); - $websiteIdsInvalidated = array_values($websiteIdsInvalidated); - Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated)); - } - - /** - * @param $optionName - * @return array|false|mixed|string - */ - private function getArrayValueFromOptionName($optionName) - { - Option::clearCachedOption($optionName); - $array = Option::get($optionName); - - if ($array - && ($array = unserialize($array)) - && count($array) - ) { - return $array; - } - return array(); - } -} \ No newline at end of file diff --git a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php index 29668098d6e..ff2acc1edf2 100644 --- a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php +++ b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php @@ -8,7 +8,6 @@ namespace Piwik\Plugins\CoreAdminHome\Tasks; use Piwik\Concurrency\DistributedList; -use Piwik\Option; /** * TODO diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index 4030969fc5f..3963690b28d 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -11,7 +11,7 @@ use Piwik\Config; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\ArchiveWriter; -use Piwik\DataAccess\InvalidatedReports; +use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\Date; use Piwik\Db; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index 4f159e83993..55faf4b3be6 100755 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -9,7 +9,7 @@ use Piwik\Common; use Piwik\DataAccess\ArchiveInvalidator; -use Piwik\DataAccess\InvalidatedReports; +use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\Db; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables; From 0509923974ef44205cae1c349fac94430705dc6e Mon Sep 17 00:00:00 2001 From: diosmosis Date: Wed, 4 Mar 2015 22:58:56 -0800 Subject: [PATCH 14/31] Refs #7181, move ArchiveInvalidator to Piwik\Archive\Invalidator away from Piwik\DataAccess since it iss not a DAO. --- core/Archive.php | 4 ++-- .../ArchiveInvalidator.php => Archive/Invalidator.php} | 8 +++++--- core/Archive/Purger.php | 2 +- core/CronArchive.php | 4 ++-- core/Tracker/Visit.php | 4 ++-- plugins/CoreAdminHome/API.php | 4 ++-- .../CoreAdminHome/Commands/FixDuplicateLogActions.php | 10 +++++----- plugins/SitesManager/SitesManager.php | 4 ++-- .../tests/Integration/SitesManagerTest.php | 4 ++-- tests/PHPUnit/Integration/CronArchiveTest.php | 4 ++-- .../Integration/DataAccess/ArchiveInvalidatorTest.php | 8 ++++---- tests/PHPUnit/Integration/Tracker/VisitTest.php | 4 ++-- ...VisitorsTwoWebsitesDifferentDaysConversionsTest.php | 4 ++-- ...sWithCustomVariablesSegmentMatchVisitorTypeTest.php | 2 +- 14 files changed, 34 insertions(+), 32 deletions(-) rename core/{DataAccess/ArchiveInvalidator.php => Archive/Invalidator.php} (98%) diff --git a/core/Archive.php b/core/Archive.php index 60d67917260..25e6c6a494c 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -10,7 +10,7 @@ use Piwik\Archive\Parameters; use Piwik\ArchiveProcessor\Rules; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\DataAccess\ArchiveSelector; use Piwik\Period\Factory as PeriodFactory; @@ -524,7 +524,7 @@ private function invalidatedReportsIfNeeded() return; // all requested site ids were already handled } - $invalidator = new ArchiveInvalidator(); + $invalidator = new Invalidator(); $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); foreach ($sitesPerDays as $date => $siteIds) { diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/Archive/Invalidator.php similarity index 98% rename from core/DataAccess/ArchiveInvalidator.php rename to core/Archive/Invalidator.php index 470dff18487..6b98061da10 100644 --- a/core/DataAccess/ArchiveInvalidator.php +++ b/core/Archive/Invalidator.php @@ -7,9 +7,11 @@ * */ -namespace Piwik\DataAccess; +namespace Piwik\Archive; use Piwik\CronArchive\SitesToReprocessDistributedList; +use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataAccess\Model; use Piwik\Date; use Piwik\Db; use Piwik\Option; @@ -27,9 +29,9 @@ * * The invalidated archives will be deleted by Purger * - * @package Piwik\DataAccess + * TODO: modify */ -class ArchiveInvalidator +class Invalidator { private $warningDates = array(); private $processedDates = array(); diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index 6f34fddeada..633f2e4bfbe 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -23,7 +23,7 @@ * * This class purges two types of archives: * - * (1) Deletes invalidated archives (from ArchiveInvalidator) + * (1) Deletes invalidated archives (from Invalidator) * * (2) Deletes outdated archives (the temporary or errored archives) * diff --git a/core/CronArchive.php b/core/CronArchive.php index 9e592e58d2f..7a970acb449 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -12,7 +12,7 @@ use Piwik\ArchiveProcessor\Rules; use Piwik\CronArchive\FixedSiteIds; use Piwik\CronArchive\SharedSiteIds; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Metrics\Formatter; use Piwik\Period\Factory as PeriodFactory; @@ -977,7 +977,7 @@ private function getApiToInvalidateArchivedReport() public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain() { - $invalidator = new ArchiveInvalidator(); + $invalidator = new Invalidator(); $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); foreach ($sitesPerDays as $date => $siteIds) { diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php index e5dbc69ef66..3a149cfbd04 100644 --- a/core/Tracker/Visit.php +++ b/core/Tracker/Visit.php @@ -11,7 +11,7 @@ use Piwik\Common; use Piwik\Config; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Date; use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Network\IPUtils; @@ -682,7 +682,7 @@ private function markArchivedReportsAsInvalidIfArchiveAlreadyFinished() $date = Date::factory((int) $time, $timezone); if (!$date->isToday()) { // we don't have to handle in case date is in future as it is not allowed by tracker - $invalidReport = new ArchiveInvalidator(); + $invalidReport = new Invalidator(); $invalidReport->rememberToInvalidateArchivedReportsLater($idSite, $date); } } diff --git a/plugins/CoreAdminHome/API.php b/plugins/CoreAdminHome/API.php index 60ba204d458..d3041a170dc 100644 --- a/plugins/CoreAdminHome/API.php +++ b/plugins/CoreAdminHome/API.php @@ -10,7 +10,7 @@ use Exception; use Piwik\Container\StaticContainer; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Db; use Piwik\Piwik; use Piwik\Scheduler\Scheduler; @@ -69,7 +69,7 @@ public function invalidateArchivedReports($idSites, $dates, $period = false) Piwik::checkUserHasAdminAccess($idSites); - $invalidator = new ArchiveInvalidator(); + $invalidator = new Invalidator(); $output = $invalidator->markArchivesAsInvalidated($idSites, $dates, $period); Site::clearCache(); diff --git a/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php b/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php index 47a058c7ad1..b26b866e27a 100644 --- a/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php +++ b/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php @@ -11,7 +11,7 @@ use Piwik\Common; use Piwik\Container\StaticContainer; use Piwik\DataAccess\Actions; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Plugin\ConsoleCommand; use Piwik\Plugins\CoreAdminHome\Model\DuplicateActionRemover; use Piwik\Timer; @@ -39,7 +39,7 @@ class FixDuplicateLogActions extends ConsoleCommand /** * Used to invalidate archives. Only used if $shouldInvalidateArchives is true. * - * @var ArchiveInvalidator + * @var Invalidator */ private $archiveInvalidator; @@ -65,17 +65,17 @@ class FixDuplicateLogActions extends ConsoleCommand /** * Constructor. * - * @param ArchiveInvalidator $invalidator + * @param Invalidator $invalidator * @param DuplicateActionRemover $duplicateActionRemover * @param Actions $actionsAccess * @param LoggerInterface $logger */ - public function __construct(ArchiveInvalidator $invalidator = null, DuplicateActionRemover $duplicateActionRemover = null, + public function __construct(Invalidator $invalidator = null, DuplicateActionRemover $duplicateActionRemover = null, Actions $actionsAccess = null, LoggerInterface $logger = null) { parent::__construct(); - $this->archiveInvalidator = $invalidator ?: new ArchiveInvalidator(); + $this->archiveInvalidator = $invalidator ?: new Invalidator(); $this->duplicateActionRemover = $duplicateActionRemover ?: new DuplicateActionRemover(); $this->actionsAccess = $actionsAccess ?: new Actions(); $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface'); diff --git a/plugins/SitesManager/SitesManager.php b/plugins/SitesManager/SitesManager.php index f06ff69948d..d88c2e7c417 100644 --- a/plugins/SitesManager/SitesManager.php +++ b/plugins/SitesManager/SitesManager.php @@ -7,7 +7,7 @@ * */ namespace Piwik\Plugins\SitesManager; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Tracker\Cache; /** @@ -38,7 +38,7 @@ public function onSiteDeleted($idSite) // we do not delete logs here on purpose (you can run these queries on the log_ tables to delete all data) Cache::deleteCacheWebsiteAttributes($idSite); - $archiveInvalidator = new ArchiveInvalidator(); + $archiveInvalidator = new Invalidator(); $archiveInvalidator->forgetRememberedArchivedReportsToInvalidateForSite($idSite); } diff --git a/plugins/SitesManager/tests/Integration/SitesManagerTest.php b/plugins/SitesManager/tests/Integration/SitesManagerTest.php index 135f505f018..9fc69fdfde1 100644 --- a/plugins/SitesManager/tests/Integration/SitesManagerTest.php +++ b/plugins/SitesManager/tests/Integration/SitesManagerTest.php @@ -10,7 +10,7 @@ use Piwik\Access; use Piwik\Cache; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Date; use Piwik\Plugins\SitesManager\SitesManager; use Piwik\Tests\Framework\Fixture; @@ -56,7 +56,7 @@ public function test_onSiteDeleted_shouldClearSiteCache() public function test_onSiteDeleted_shouldRemoveRememberedArchiveReports() { - $archive = new ArchiveInvalidator(); + $archive = new Invalidator(); $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-05')); $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-06')); $archive->rememberToInvalidateArchivedReportsLater(4949, Date::factory('2014-04-05')); diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php index a85f227802b..9397fe511cb 100644 --- a/tests/PHPUnit/Integration/CronArchiveTest.php +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -9,7 +9,7 @@ namespace Piwik\Tests\Integration; use Piwik\CronArchive; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Date; use Piwik\Db; use Piwik\Plugins\CoreAdminHome\tests\Framework\Mock\API; @@ -46,7 +46,7 @@ public function setUp() public function test_getColumnNamesFromTable() { - $ar = new ArchiveInvalidator(); + $ar = new Invalidator(); $ar->rememberToInvalidateArchivedReportsLater(1, Date::factory('2014-04-05')); $ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-05')); $ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-06')); diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index fbf2c0b7a94..24e178b1537 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -11,17 +11,17 @@ use Piwik\Date; use Piwik\Option; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; /** * @group Archiver - * @group ArchiveInvalidator + * @group Invalidator * @group DataAccess */ class ArchiveInvalidatorTest extends IntegrationTestCase { /** - * @var ArchiveInvalidator + * @var Invalidator */ private $invalidator; @@ -29,7 +29,7 @@ public function setUp() { parent::setUp(); - $this->invalidator = new ArchiveInvalidator(); + $this->invalidator = new Invalidator(); } public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet() diff --git a/tests/PHPUnit/Integration/Tracker/VisitTest.php b/tests/PHPUnit/Integration/Tracker/VisitTest.php index bbbb972cc3e..ed767b1ea0e 100644 --- a/tests/PHPUnit/Integration/Tracker/VisitTest.php +++ b/tests/PHPUnit/Integration/Tracker/VisitTest.php @@ -11,7 +11,7 @@ use Piwik\Access; use Piwik\Cache; use Piwik\CacheId; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Date; use Piwik\Network\IPUtils; use Piwik\Plugin\Manager; @@ -370,7 +370,7 @@ private function assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $visit->handle(); - $archive = new ArchiveInvalidator(); + $archive = new Invalidator(); $remembered = $archive->getRememberedArchivedReportsThatShouldBeInvalidated(); $this->assertSame($expectedRemeberedArchivedReports, $remembered); diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php index 56dc98e9a06..3c1509f0c6a 100755 --- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php +++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php @@ -9,7 +9,7 @@ use Piwik\Archive; use Piwik\Cache; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\Option; use Piwik\Plugins\Goals\Archiver; use Piwik\Segment; @@ -172,7 +172,7 @@ public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePer $this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2), $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated')); - $invalidator = new ArchiveInvalidator(); + $invalidator = new Invalidator(); self::$fixture->trackVisits(); diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index 55faf4b3be6..4a74c954bc1 100755 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -8,7 +8,7 @@ namespace Piwik\Tests\System; use Piwik\Common; -use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Archive\Invalidator; use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\Db; use Piwik\Tests\Framework\TestCase\SystemTestCase; From 51984c0bbff0a55d6ff6801f09546dc6b417be1a Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 00:30:11 -0800 Subject: [PATCH 15/31] Refs #7181, fill in TODO docs. --- core/Archive/Invalidator.php | 22 +++++++++++++--- core/Archive/Purger.php | 25 +++++++++++-------- core/Concurrency/DistributedList.php | 23 ++++++++++------- .../SitesToReprocessDistributedList.php | 11 +++----- core/DataAccess/Model.php | 5 ++-- .../Commands/PurgeOldArchiveData.php | 3 ++- .../Tasks/ArchivesToPurgeDistributedList.php | 18 +++++++++---- 7 files changed, 68 insertions(+), 39 deletions(-) diff --git a/core/Archive/Invalidator.php b/core/Archive/Invalidator.php index 6b98061da10..bdb4dd9bf61 100644 --- a/core/Archive/Invalidator.php +++ b/core/Archive/Invalidator.php @@ -23,13 +23,27 @@ use Piwik\Site; /** - * Marks archives as Invalidated by setting the done flag to a special value (see Model->updateArchiveAsInvalidated) + * Service that can be used to invalidate archives or add archive references to a list so they will + * be invalidated later. * - * Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive) + * Archives are put in an "invalidated" state by setting the done flag to `ArchiveWriter::DONE_INVALIDATED`. + * This class also adds the archive's associated site to the a distributed list and adding the archive's year month to another + * distributed list. * - * The invalidated archives will be deleted by Purger + * CronArchive will reprocess the archive data for all sites in the first list, and a scheduled task + * will purge the old, invalidated data in archive tables identified by the second list. * - * TODO: modify + * Until CronArchive, or browser triggered archiving, re-processes data for an invalidated archive, the invalidated + * archive data will still be displayed in the UI and API. + * + * ### Deferred Invalidation + * + * Invalidating archives means running queries on one or more archive tables. In some situations, like during + * tracking, this is not desired. In such cases, archive references can be added to a list via the + * rememberToInvalidateArchivedReportsLater method, which will add the reference to a distributed list + * + * Later, during Piwik's normal execution, the list will be read and every archive it references will + * be invalidated. */ class Invalidator { diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index 633f2e4bfbe..aca9d59ad6c 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -11,7 +11,6 @@ use Piwik\ArchiveProcessor\Rules; use Piwik\Config; use Piwik\DataAccess\ArchiveTableCreator; -use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\DataAccess\Model; use Piwik\Date; use Piwik\Db; @@ -20,13 +19,15 @@ use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; /** + * Service that purges temporary, error-ed, invalid and custom range archives from archive tables. * - * This class purges two types of archives: + * Temporary archives are purged if they were archived before a specific time. The time is dependent + * on whether browser triggered archiving is enabled or not. * - * (1) Deletes invalidated archives (from Invalidator) - * - * (2) Deletes outdated archives (the temporary or errored archives) + * Error-ed archives are purged w/o constraint. * + * Invalid archives are purged if a new, valid, archive exists w/ the same site, date, period combination. + * Archives are marked as invalid via Piwik\Archive\Invalidator. */ class Purger { @@ -36,28 +37,29 @@ class Purger private $model; /** - * TODO + * Date threshold for purging custom range archives. Archives that are older than this date + * are purged unconditionally from the requested archive table. * * @var Date */ private $purgeCustomRangesOlderThan; /** - * TODO + * Date to use for 'yesterday'. Exists so tests can override this value. * * @var Date */ private $yesterday; /** - * TODO + * Date to use for 'today'. Exists so tests can override this value. * * @var $today */ private $today; /** - * TODO + * Date to use for 'now'. Exists so tests can override this value. * * @var int */ @@ -96,7 +98,10 @@ public function purgeInvalidatedArchives() } /** - * TODO + * Purge all invalidate archives for whom there are newer, valid archives from the archive + * table that stores data for `$date`. + * + * @param Date $date The date identifying the archive table. */ public function purgeInvalidatedArchivesFrom(Date $date) { diff --git a/core/Concurrency/DistributedList.php b/core/Concurrency/DistributedList.php index 2f37fe0e4bf..3823b4b47f7 100644 --- a/core/Concurrency/DistributedList.php +++ b/core/Concurrency/DistributedList.php @@ -10,21 +10,27 @@ use Piwik\Option; /** - * TODO + * Manages a simple distributed list stored in an Option. No locking occurs, so the list + * is not thread safe, and should only be used for use cases where atomicity is not + * important. * - * TODO: tests + * The list of items is serialized and stored in an Option. Items are converted to string + * before being persisted, so it is not expected to unserialize objects. + * TODO: light integration tests */ class DistributedList { /** - * TODO + * The name of the option to store the list in. * * @var string */ private $optionName; /** - * TODO + * Constructor. + * + * @param string $optionName */ public function __construct($optionName) { @@ -32,7 +38,7 @@ public function __construct($optionName) } /** - * TODO + * Queries the option table and returns all items in this list. * * @return array */ @@ -51,7 +57,7 @@ public function getAll() } /** - * TODO + * Sets the contents of the list in the option table. * * @param string[] $items */ @@ -65,7 +71,7 @@ public function setAll($items) } /** - * TODO + * Adds one or more items to the list in the option table. * * @param string|array $item */ @@ -82,8 +88,7 @@ public function add($item) } /** - * TODO - * TODO: support removing multiple + * Removes one or more items by value from the list in the optiontable. * * @param string|array $items */ diff --git a/core/CronArchive/SitesToReprocessDistributedList.php b/core/CronArchive/SitesToReprocessDistributedList.php index c16062105ee..2ec471aec58 100644 --- a/core/CronArchive/SitesToReprocessDistributedList.php +++ b/core/CronArchive/SitesToReprocessDistributedList.php @@ -11,15 +11,12 @@ use Piwik\Concurrency\DistributedList; /** - * Keeps track of which reports were invalidated via CoreAdminHome.invalidateArchivedReports API. + * Distributed list that stores the list of IDs of sites whose archives should be reprocessed. * - * This is used by: + * CronArchive will read this list of sites when archiving is being run, and make sure the sites + * are re-archived. * - * 1. core:archive command to know which websites should be reprocessed - * - * 2. scheduled task purgeInvalidatedArchives to know which websites/months should be purged - * - * TODO: modify + * Any class/API method/command/etc. is allowed to add site IDs to this list. */ class SitesToReprocessDistributedList extends DistributedList { diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index a8fc0ff4a0a..0980983fe31 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -246,10 +246,9 @@ public function insertRecord($tableName, $fields, $record, $name, $value) } /** - * TODO - * TODO: tests? there doesn't seem to be tests for other methods here. + * Returns the site IDs for invalidated archives in an archive table. * - * @param string $numericTable + * @param string $numericTable The numeric table to search through. * @return int[] */ public function getSitesWithInvalidatedArchive($numericTable) diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php index 23884736a0f..29b1633927d 100644 --- a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php +++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php @@ -21,7 +21,8 @@ use Symfony\Component\Console\Output\OutputInterface; /** - * TODO + * Command that allows users to force purge old or invalid archive data. In the event of a failure + * in the archive purging scheduled task, this command can be used to manually delete old/invalid archives. * TODO: command tests */ class PurgeOldArchiveData extends ConsoleCommand diff --git a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php index ff2acc1edf2..a1a28de369c 100644 --- a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php +++ b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php @@ -10,14 +10,24 @@ use Piwik\Concurrency\DistributedList; /** - * TODO + * Distributed list that holds a list of year-month archive table identifiers (eg, 2015_01 or 2014_11). Each item in the + * list is expected to identify a pair of archive tables that contain invalidated archives. + * + * The archiving purging scheduled task will read items in this list when executing the daily purge. + * + * This class is necessary in order to keep the archive purging scheduled task fast. W/o a way to keep track of + * tables w/ invalid data, the task would have to iterate over every table, which is not desired for a task that + * is executed daily. + * + * If users find other tables contain invalidated archives, they can use the core:purge-old-archive-data command + * to manually purge them. */ class ArchivesToPurgeDistributedList extends DistributedList { const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds'; /** - * TODO + * Constructor. */ public function __construct() { @@ -25,9 +35,7 @@ public function __construct() } /** - * TODO - * - * @param array $yearMonths + * @inheritdoc */ public function setAll($yearMonths) { From d7380293fc158543a6bb3e88d9e2ce6da1dc7ff1 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 00:44:55 -0800 Subject: [PATCH 16/31] Refs #7181, move distributed list access from Archive\Purger to CoreAdminHome's scheduled tasks so Piwik core is no longer dependent on it. --- core/Archive/Purger.php | 21 ------------------ plugins/CoreAdminHome/Tasks.php | 9 +++++++- .../Tasks/ArchivesToPurgeDistributedList.php | 22 +++++++++++++++++++ 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index aca9d59ad6c..3d91170a37f 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -76,27 +76,6 @@ public function __construct(Model $model = null, Date $purgeCustomRangesOlderTha $this->now = time(); } - /** - * TODO - */ - public function purgeInvalidatedArchives() - { - $archivesToPurge = new ArchivesToPurgeDistributedList(); - - $yearMonths = $archivesToPurge->getAll(); - foreach ($yearMonths as $yearMonth) { - try { - $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); - } catch (\Exception $ex) { - continue; // invalid year month in distributed list - } - - $this->purgeInvalidatedArchivesFrom($date); - - $archivesToPurge->remove($yearMonth); - } - } - /** * Purge all invalidate archives for whom there are newer, valid archives from the archive * table that stores data for `$date`. diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 50b547cb62d..e09d9ad03ce 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -14,7 +14,9 @@ use Piwik\Date; use Piwik\Db; use Piwik\Log; +use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; +// TODO: tests for Tasks class Tasks extends \Piwik\Plugin\Tasks { /** @@ -64,7 +66,12 @@ public function purgeOutdatedArchives() public function purgeInvalidatedArchives() { - $this->archivePurger->purgeInvalidatedArchives(); + $archivesToPurge = new ArchivesToPurgeDistributedList(); + foreach ($archivesToPurge->getAllAsDates() as $date) { + $this->archivePurger->purgeInvalidatedArchivesFrom($date); + + $archivesToPurge->removeDate($date); + } } public function optimizeArchiveTable() diff --git a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php index a1a28de369c..a7c9b6eb9ab 100644 --- a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php +++ b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php @@ -8,6 +8,7 @@ namespace Piwik\Plugins\CoreAdminHome\Tasks; use Piwik\Concurrency\DistributedList; +use Piwik\Date; /** * Distributed list that holds a list of year-month archive table identifiers (eg, 2015_01 or 2014_11). Each item in the @@ -42,4 +43,25 @@ public function setAll($yearMonths) $yearMonths = array_unique($yearMonths); parent::setAll($yearMonths); } + + public function getAllAsDates() + { + $dates = array(); + foreach ($this->getAll() as $yearMonth) { + try { + $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); + } catch (\Exception $ex) { + continue; // invalid year month in distributed list + } + + $dates[] = $date; + } + return $dates; + } + + public function removeDate(Date $date) + { + $yearMonth = $date->toString('Y_m'); + $this->remove($yearMonth); + } } \ No newline at end of file From 7c3c4057d2d540025e4da19ee4c82b06f163d07e Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 03:22:58 -0800 Subject: [PATCH 17/31] Refs #7181, move ArchivesToPurgeDistributedList to CoreAdminHome since only consumer for it is the scheduled task and add tests for task in CoreAdminHome (or rather move test from PurgerTest to new TasksTest class). Moved test setup to new fixture type. --- plugins/CoreAdminHome/Tasks.php | 1 - .../tests/Integration/TasksTest.php | 88 ++++ .../RawArchiveDataWithTempAndInvalidated.php | 387 +++++++++++++++++ .../Integration/Archive/PurgerTest.php | 404 +----------------- 4 files changed, 492 insertions(+), 388 deletions(-) create mode 100644 plugins/CoreAdminHome/tests/Integration/TasksTest.php create mode 100644 tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index e09d9ad03ce..07398a83e34 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -16,7 +16,6 @@ use Piwik\Log; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; -// TODO: tests for Tasks class Tasks extends \Piwik\Plugin\Tasks { /** diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php new file mode 100644 index 00000000000..aacc20d7f7b --- /dev/null +++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php @@ -0,0 +1,88 @@ +january = Date::factory('2015-01-01'); + $this->february = Date::factory('2015-02-01'); + + $archivePurger = new Purger(); + $archivePurger->setTodayDate(Date::factory('2015-02-27')); + $archivePurger->setYesterdayDate(Date::factory('2015-02-26')); + $archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp()); + + $this->tasks = new Tasks($archivePurger); + } + + public function test_purgeInvalidatedArchives_PurgesCorrectInvalidatedArchives_AndOnlyPurgesDataForDatesAndSites_InInvalidatedReportsDistributedList() + { + $this->setUpInvalidatedReportsDistributedList($dates = array($this->february)); + + $this->tasks->purgeInvalidatedArchives(); + + self::$fixture->assertFebruaryInvalidatedArchivesPurged(); + self::$fixture->assertJanuaryInvalidatedArchivesNotPurged(); + + // assert invalidated reports distributed list has changed + $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList(); + $yearMonths = $archivesToPurgeDistributedList->getAll(); + + $this->assertEmpty($yearMonths); + } + + /** + * @param Date[] $dates + */ + private function setUpInvalidatedReportsDistributedList($dates) + { + $yearMonths = array(); + foreach ($dates as $date) { + $yearMonths[] = $date->toString('Y_m'); + } + + $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList(); + $archivesToPurgeDistributedList->add($yearMonths); + } +} + +TasksTest::$fixture = new RawArchiveDataWithTempAndInvalidated(); \ No newline at end of file diff --git a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php new file mode 100644 index 00000000000..080aa3c0431 --- /dev/null +++ b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php @@ -0,0 +1,387 @@ + 1, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-03', + 'date2' => '2015-02-03', + 'period' => 1, + 'ts_archived' => '2015-02-03 12:12:12' + ), + + array( + 'idarchive' => 2, + 'idsite' => 2, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-01', + 'date2' => '2015-02-31', + 'period' => 3, + 'ts_archived' => '2015-02-18 10:10:10' + ), + + array( + 'idarchive' => 3, + 'idsite' => 3, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-04', + 'date2' => '2015-02-10', + 'period' => 2, + 'ts_archived' => '2015-02-10 12:34:56' + ), + + array( + 'idarchive' => 4, + 'idsite' => 1, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-15', + 'date2' => '2015-02-15', + 'period' => 1, + 'ts_archived' => '2015-02-15 08:12:13' + ), + + + // valid temporary + array( // only valid + 'idarchive' => 5, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-27', + 'date2' => '2015-02-27', + 'period' => 1, + 'ts_archived' => '2015-02-27 08:08:08' + ), + + array( + 'idarchive' => 6, + 'idsite' => 2, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-26', + 'date2' => '2015-02-26', + 'period' => 1, + 'ts_archived' => '2015-02-26 07:07:07' + ), + + array( + 'idarchive' => 7, + 'idsite' => 3, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-01', + 'date2' => '2015-02-28', + 'period' => 3, + 'ts_archived' => '2015-02-15 00:00:00' + ), + + // custom ranges + array( + 'idarchive' => 8, + 'idsite' => 1, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-03', + 'date2' => '2015-02-14', + 'period' => 5, + 'ts_archived' => '2015-02-27 00:00:00' + ), + + array( + 'idarchive' => 9, + 'idsite' => 2, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-05', + 'date2' => '2015-02-14', + 'period' => 5, + 'ts_archived' => '2015-02-15 00:00:00' + ), + + array( + 'idarchive' => 10, + 'idsite' => 3, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-05', + 'date2' => '2015-03-05', + 'period' => 5, + 'ts_archived' => '2015-02-26 00:00:00' + ), + + // invalidated + array( + 'idarchive' => 11, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-10', + 'date2' => '2015-02-10', + 'period' => 1, + 'ts_archived' => '2015-02-10 12:13:14' + ), + + array( + 'idarchive' => 12, + 'idsite' => 2, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-08', + 'date2' => '2015-02-14', + 'period' => 2, + 'ts_archived' => '2015-02-15 00:00:00' + ), + + array( + 'idarchive' => 13, + 'idsite' => 3, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-01', + 'date2' => '2015-02-28', + 'period' => 3, + 'ts_archived' => '2015-02-27 13:13:13' + ), + + array( + 'idarchive' => 14, + 'idsite' => 1, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-28', + 'date2' => '2015-02-28', + 'period' => 1, + 'ts_archived' => '2015-02-28 12:12:12' + ), + + array( + 'idarchive' => 15, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_INVALIDATED, + 'date1' => '2015-02-27', + 'date2' => '2015-02-27', + 'period' => 1, + 'ts_archived' => '2015-02-28 12:12:12' + ), + + // reprocessed invalidated + array( + 'idarchive' => 16, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-10', + 'date2' => '2015-02-10', + 'period' => 1, + 'ts_archived' => '2015-02-11 12:13:14' + ), + + array( + 'idarchive' => 17, + 'idsite' => 2, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-08', + 'date2' => '2015-02-14', + 'period' => 2, + 'ts_archived' => '2015-02-16 00:00:00' + ), + + array( + 'idarchive' => 18, + 'idsite' => 3, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_OK, + 'date1' => '2015-02-01', + 'date2' => '2015-02-28', + 'period' => 3, + 'ts_archived' => '2015-02-28 13:13:13' + ), + + array( + 'idarchive' => 19, + 'idsite' => 1, + 'name' => 'doneDUMMYHASHSTR', + 'value' => ArchiveWriter::DONE_OK_TEMPORARY, + 'date1' => '2015-02-28', + 'date2' => '2015-02-28', + 'period' => 1, + 'ts_archived' => '2015-02-28 16:12:12' // must be late so it doesn't screw up the purgeOutdatedArchives test + ), + ); + + /** + * @var Date + */ + public $january; + + /** + * @var Date + */ + public $february; + + public function setUp() + { + parent::setUp(); + + $this->january = Date::factory('2015-01-01'); + $this->february = Date::factory('2015-02-01'); + + $this->insertOutdatedArchives($this->january); + $this->insertOutdatedArchives($this->february); + } + + private function insertOutdatedArchives(Date $archiveDate) + { + $dummyArchiveData = $this->getDummyArchiveDataForDate($archiveDate); + + $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); + foreach ($dummyArchiveData as $row) { + // done row + $this->insertTestArchiveRow($numericTable, $row); + + // two metrics + $row['name'] = 'nb_visits'; + $row['value'] = 1; + $this->insertTestArchiveRow($numericTable, $row); + + $row['name'] = 'nb_actions'; + $row['value'] = 2; + $this->insertTestArchiveRow($numericTable, $row); + } + + $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); + foreach ($dummyArchiveData as $row) { + // two blobs + $row['name'] = 'blobname'; + $row['value'] = 'dummyvalue'; + $this->insertTestArchiveRow($blobTable, $row); + + $row['name'] = 'blobname2'; + $row['value'] = 'dummyvalue'; + $this->insertTestArchiveRow($blobTable, $row); + } + } + + private function insertTestArchiveRow($table, $row) + { + $insertSqlTemplate = "INSERT INTO %s (idarchive, idsite, name, value, date1, date2, period, ts_archived) VALUES ('%s')"; + + Db::exec(sprintf($insertSqlTemplate, $table, implode("','", $row))); + } + + private function getDummyArchiveDataForDate($archiveDate) + { + $rows = self::$dummyArchiveData; + foreach ($rows as &$row) { + $row['date1'] = $this->setDateMonthAndYear($row['date1'], $archiveDate); + $row['date2'] = $this->setDateMonthAndYear($row['date1'], $archiveDate); + } + return$rows; + } + + private function setDateMonthAndYear($dateString, Date $archiveDate) + { + return $archiveDate->toString('Y-m') . '-' . Date::factory($dateString)->toString('d'); + } + + public function assertFebruaryTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled) + { + if ($isBrowserTriggeredArchivingEnabled) { + $expectedPurgedArchives = array(1,2,3,4,6,7); // only archives from 2 hours before "now" are purged + } else { + $expectedPurgedArchives = array(1,2,3,4,7); // only archives before start of "yesterday" are purged + } + + $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + } + + public function assertFebruaryCustomRangesPurged() + { + $expectedPurgedArchives = array(8,9,10); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + } + + public function assertJanuaryTemporaryArchivesNotPurged() + { + $expectedPresentArchives = array(1,2,3,4,5,6,7); + $this->assertArchivesExist($expectedPresentArchives, $this->january); + } + + public function assertJanuaryInvalidatedArchivesNotPurged() + { + $expectedPresentArchives = array(11, 12, 13, 14); + $this->assertArchivesExist($expectedPresentArchives, $this->january); + } + + public function assertArchivesDoNotExist($expectedPurgedArchiveIds, $archiveDate) + { + $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); + $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); + + $numericPurgedArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPurgedArchiveIds); + $this->assertEquals(0, $numericPurgedArchiveCount); + + $blobPurgedArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPurgedArchiveIds); + $this->assertEquals(0, $blobPurgedArchiveCount); + } + + public function assertArchivesExist($expectedPresentArchiveIds, $archiveDate) + { + $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); + $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); + + $numericArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPresentArchiveIds); + $expectedNumericRowCount = count($expectedPresentArchiveIds) * 3; // two metrics + 1 done row + $this->assertEquals($expectedNumericRowCount, $numericArchiveCount); + + $blobArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPresentArchiveIds); + $expectedBlobRowCount = count($expectedPresentArchiveIds) * 2; // two blob rows + $this->assertEquals($expectedBlobRowCount, $blobArchiveCount); + } + + private function getArchiveRowCountWithId($table, $archiveIds) + { + return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE idarchive IN (".implode(',', $archiveIds).")"); + } + + public function assertFebruaryInvalidatedArchivesPurged() + { + // check invalidated archives for all sites are purged + $expectedPurgedArchives = array(11, 12, 13, 14); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + + // check archive 15 is not purged since it doesn't have newer DONE_OK/DONE_TEMPORARY archive + $expectedExistingArchives = array(15); + $this->assertArchivesExist($expectedExistingArchives, $this->february); + } +} \ No newline at end of file diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index 3963690b28d..2f27127e718 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -10,11 +10,9 @@ use Piwik\Archive\Purger; use Piwik\Config; use Piwik\DataAccess\ArchiveTableCreator; -use Piwik\DataAccess\ArchiveWriter; -use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\Date; use Piwik\Db; -use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; +use Piwik\Tests\Fixtures\RawArchiveDataWithTempAndInvalidated; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; /** @@ -26,222 +24,10 @@ class PurgerTest extends IntegrationTestCase * - that archive purging is done daily (when called through cron archiving and through API request for scheduled task running) [SYSTEM TESTS] */ - private static $dummyArchiveData = array( - // outdated temporary - array( - 'idarchive' => 1, - 'idsite' => 1, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-03', - 'date2' => '2015-02-03', - 'period' => 1, - 'ts_archived' => '2015-02-03 12:12:12' - ), - - array( - 'idarchive' => 2, - 'idsite' => 2, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-01', - 'date2' => '2015-02-31', - 'period' => 3, - 'ts_archived' => '2015-02-18 10:10:10' - ), - - array( - 'idarchive' => 3, - 'idsite' => 3, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-04', - 'date2' => '2015-02-10', - 'period' => 2, - 'ts_archived' => '2015-02-10 12:34:56' - ), - - array( - 'idarchive' => 4, - 'idsite' => 1, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-15', - 'date2' => '2015-02-15', - 'period' => 1, - 'ts_archived' => '2015-02-15 08:12:13' - ), - - - // valid temporary - array( // only valid - 'idarchive' => 5, - 'idsite' => 1, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-27', - 'date2' => '2015-02-27', - 'period' => 1, - 'ts_archived' => '2015-02-27 08:08:08' - ), - - array( - 'idarchive' => 6, - 'idsite' => 2, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-26', - 'date2' => '2015-02-26', - 'period' => 1, - 'ts_archived' => '2015-02-26 07:07:07' - ), - - array( - 'idarchive' => 7, - 'idsite' => 3, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-01', - 'date2' => '2015-02-28', - 'period' => 3, - 'ts_archived' => '2015-02-15 00:00:00' - ), - - // custom ranges - array( - 'idarchive' => 8, - 'idsite' => 1, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_OK, - 'date1' => '2015-02-03', - 'date2' => '2015-02-14', - 'period' => 5, - 'ts_archived' => '2015-02-27 00:00:00' - ), - - array( - 'idarchive' => 9, - 'idsite' => 2, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_OK, - 'date1' => '2015-02-05', - 'date2' => '2015-02-14', - 'period' => 5, - 'ts_archived' => '2015-02-15 00:00:00' - ), - - array( - 'idarchive' => 10, - 'idsite' => 3, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-05', - 'date2' => '2015-03-05', - 'period' => 5, - 'ts_archived' => '2015-02-26 00:00:00' - ), - - // invalidated - array( - 'idarchive' => 11, - 'idsite' => 1, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_INVALIDATED, - 'date1' => '2015-02-10', - 'date2' => '2015-02-10', - 'period' => 1, - 'ts_archived' => '2015-02-10 12:13:14' - ), - - array( - 'idarchive' => 12, - 'idsite' => 2, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_INVALIDATED, - 'date1' => '2015-02-08', - 'date2' => '2015-02-14', - 'period' => 2, - 'ts_archived' => '2015-02-15 00:00:00' - ), - - array( - 'idarchive' => 13, - 'idsite' => 3, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_INVALIDATED, - 'date1' => '2015-02-01', - 'date2' => '2015-02-28', - 'period' => 3, - 'ts_archived' => '2015-02-27 13:13:13' - ), - - array( - 'idarchive' => 14, - 'idsite' => 1, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_INVALIDATED, - 'date1' => '2015-02-28', - 'date2' => '2015-02-28', - 'period' => 1, - 'ts_archived' => '2015-02-28 12:12:12' - ), - - array( - 'idarchive' => 15, - 'idsite' => 1, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_INVALIDATED, - 'date1' => '2015-02-27', - 'date2' => '2015-02-27', - 'period' => 1, - 'ts_archived' => '2015-02-28 12:12:12' - ), - - // reprocessed invalidated - array( - 'idarchive' => 16, - 'idsite' => 1, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_OK, - 'date1' => '2015-02-10', - 'date2' => '2015-02-10', - 'period' => 1, - 'ts_archived' => '2015-02-11 12:13:14' - ), - - array( - 'idarchive' => 17, - 'idsite' => 2, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_OK, - 'date1' => '2015-02-08', - 'date2' => '2015-02-14', - 'period' => 2, - 'ts_archived' => '2015-02-16 00:00:00' - ), - - array( - 'idarchive' => 18, - 'idsite' => 3, - 'name' => 'done', - 'value' => ArchiveWriter::DONE_OK, - 'date1' => '2015-02-01', - 'date2' => '2015-02-28', - 'period' => 3, - 'ts_archived' => '2015-02-28 13:13:13' - ), - - array( - 'idarchive' => 19, - 'idsite' => 1, - 'name' => 'doneDUMMYHASHSTR', - 'value' => ArchiveWriter::DONE_OK_TEMPORARY, - 'date1' => '2015-02-28', - 'date2' => '2015-02-28', - 'period' => 1, - 'ts_archived' => '2015-02-28 16:12:12' // must be late so it doesn't screw up the purgeOutdatedArchives test - ), - ); + /** + * @var RawArchiveDataWithTempAndInvalidated + */ + public static $fixture; /** * @var Purger @@ -262,17 +48,14 @@ public function setUp() { parent::setUp(); - $this->january = Date::factory('2015-01-01'); - $this->february = Date::factory('2015-02-01'); + $this->january = self::$fixture->january; + $this->february = self::$fixture->february; $this->archivePurger = new Purger(); $this->archivePurger->setTodayDate(Date::factory('2015-02-27')); $this->archivePurger->setYesterdayDate(Date::factory('2015-02-26')); $this->archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp()); - $this->insertOutdatedArchives($this->january); - $this->insertOutdatedArchives($this->february); - $this->configureCustomRangePurging(); } @@ -282,10 +65,10 @@ public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRan $this->archivePurger->purgeOutdatedArchives($this->february); - $this->assertFebruaryTemporaryArchivesPurged($browserTriggeringEnabled = true); - $this->assertFebruaryCustomRangesPurged(); + self::$fixture->assertFebruaryTemporaryArchivesPurged($browserTriggeringEnabled = true); + self::$fixture->assertFebruaryCustomRangesPurged(); - $this->assertJanuaryTemporaryArchivesNotPurged(); + self::$fixture->assertJanuaryTemporaryArchivesNotPurged(); } public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRangeArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringDisabled() @@ -294,34 +77,18 @@ public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRan $this->archivePurger->purgeOutdatedArchives($this->february); - $this->assertFebruaryTemporaryArchivesPurged($browserTriggeringEnabled = false); - $this->assertFebruaryCustomRangesPurged(); + self::$fixture->assertFebruaryTemporaryArchivesPurged($browserTriggeringEnabled = false); + self::$fixture->assertFebruaryCustomRangesPurged(); - $this->assertJanuaryTemporaryArchivesNotPurged(); - } - - public function test_purgeInvalidatedArchives_PurgesCorrectInvalidatedArchives_AndOnlyPurgesDataForDatesAndSites_InInvalidatedReportsDistributedList() - { - $this->setUpInvalidatedReportsDistributedList($dates = array($this->february)); - - $this->archivePurger->purgeInvalidatedArchives(); - - $this->assertFebruaryInvalidatedArchivesPurged(); - $this->assertJanuaryInvalidatedArchivesNotPurged(); - - // assert invalidated reports distributed list has changed - $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList(); - $yearMonths = $archivesToPurgeDistributedList->getAll(); - - $this->assertEmpty($yearMonths); + self::$fixture->assertJanuaryTemporaryArchivesNotPurged(); } public function test_purgeInvalidatedArchivesFrom_PurgesAllInvalidatedArchives_AndMarksDatesAndSitesAsInvalidated() { $this->archivePurger->purgeInvalidatedArchivesFrom($this->february); - $this->assertFebruaryInvalidatedArchivesPurged(); - $this->assertJanuaryInvalidatedArchivesNotPurged(); + self::$fixture->assertFebruaryInvalidatedArchivesPurged(); + self::$fixture->assertJanuaryInvalidatedArchivesNotPurged(); } private function configureCustomRangePurging() @@ -338,143 +105,6 @@ private function disableBrowserTriggeredArchiving() { Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; } - - /** - * @param Date[] $dates - */ - private function setUpInvalidatedReportsDistributedList($dates) - { - $yearMonths = array(); - foreach ($dates as $date) { - $yearMonths[] = $date->toString('Y_m'); - } - - $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList(); - $archivesToPurgeDistributedList->add($yearMonths); - } - - private function insertOutdatedArchives(Date $archiveDate) - { - $dummyArchiveData = $this->getDummyArchiveDataForDate($archiveDate); - - $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); - foreach ($dummyArchiveData as $row) { - // done row - $this->insertTestArchiveRow($numericTable, $row); - - // two metrics - $row['name'] = 'nb_visits'; - $row['value'] = 1; - $this->insertTestArchiveRow($numericTable, $row); - - $row['name'] = 'nb_actions'; - $row['value'] = 2; - $this->insertTestArchiveRow($numericTable, $row); - } - - $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); - foreach ($dummyArchiveData as $row) { - // two blobs - $row['name'] = 'blobname'; - $row['value'] = 'dummyvalue'; - $this->insertTestArchiveRow($blobTable, $row); - - $row['name'] = 'blobname2'; - $row['value'] = 'dummyvalue'; - $this->insertTestArchiveRow($blobTable, $row); - } - } - - private function insertTestArchiveRow($table, $row) - { - $insertSqlTemplate = "INSERT INTO %s (idarchive, idsite, name, value, date1, date2, period, ts_archived) VALUES ('%s')"; - - Db::exec(sprintf($insertSqlTemplate, $table, implode("','", $row))); - } - - private function getDummyArchiveDataForDate($archiveDate) - { - $rows = self::$dummyArchiveData; - foreach ($rows as &$row) { - $row['date1'] = $this->setDateMonthAndYear($row['date1'], $archiveDate); - $row['date2'] = $this->setDateMonthAndYear($row['date1'], $archiveDate); - } - return$rows; - } - - private function setDateMonthAndYear($dateString, Date $archiveDate) - { - return $archiveDate->toString('Y-m') . '-' . Date::factory($dateString)->toString('d'); - } - - private function assertFebruaryTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled) - { - if ($isBrowserTriggeredArchivingEnabled) { - $expectedPurgedArchives = array(1,2,3,4,6,7); // only archives from 2 hours before "now" are purged - } else { - $expectedPurgedArchives = array(1,2,3,4,7); // only archives before start of "yesterday" are purged - } - - $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); - } - - private function assertFebruaryCustomRangesPurged() - { - $expectedPurgedArchives = array(8,9,10); - $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); - } - - private function assertJanuaryTemporaryArchivesNotPurged() - { - $expectedPresentArchives = array(1,2,3,4,5,6,7); - $this->assertArchivesExist($expectedPresentArchives, $this->january); - } - - private function assertJanuaryInvalidatedArchivesNotPurged() - { - $expectedPresentArchives = array(11, 12, 13, 14); - $this->assertArchivesExist($expectedPresentArchives, $this->january); - } - - private function assertArchivesDoNotExist($expectedPurgedArchiveIds, $archiveDate) - { - $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); - $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); - - $numericPurgedArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPurgedArchiveIds); - $this->assertEquals(0, $numericPurgedArchiveCount); - - $blobPurgedArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPurgedArchiveIds); - $this->assertEquals(0, $blobPurgedArchiveCount); - } - - private function assertArchivesExist($expectedPresentArchiveIds, $archiveDate) - { - $numericTable = ArchiveTableCreator::getNumericTable($archiveDate); - $blobTable = ArchiveTableCreator::getBlobTable($archiveDate); - - $numericArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPresentArchiveIds); - $expectedNumericRowCount = count($expectedPresentArchiveIds) * 3; // two metrics + 1 done row - $this->assertEquals($expectedNumericRowCount, $numericArchiveCount); - - $blobArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPresentArchiveIds); - $expectedBlobRowCount = count($expectedPresentArchiveIds) * 2; // two blob rows - $this->assertEquals($expectedBlobRowCount, $blobArchiveCount); - } - - private function getArchiveRowCountWithId($table, $archiveIds) - { - return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE idarchive IN (".implode(',', $archiveIds).")"); - } - - private function assertFebruaryInvalidatedArchivesPurged() - { - // check invalidated archives for all sites are purged - $expectedPurgedArchives = array(11, 12, 13, 14); - $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); - - // check archive 15 is not purged since it doesn't have newer DONE_OK/DONE_TEMPORARY archive - $expectedExistingArchives = array(15); - $this->assertArchivesExist($expectedExistingArchives, $this->february); - } } + +PurgerTest::$fixture = new RawArchiveDataWithTempAndInvalidated(); \ No newline at end of file From 1479c500f04cc921ee93f0ea89d9f8b1505f6fee Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 17:42:10 -0800 Subject: [PATCH 18/31] Refs #7181, add tests for DistributedList base type and add removeByIndex method to list for future use. --- core/Concurrency/DistributedList.php | 30 +++- .../Concurrency/DistributedListTest.php | 153 ++++++++++++++++++ 2 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 tests/PHPUnit/Integration/Concurrency/DistributedListTest.php diff --git a/core/Concurrency/DistributedList.php b/core/Concurrency/DistributedList.php index 3823b4b47f7..eecfab6a554 100644 --- a/core/Concurrency/DistributedList.php +++ b/core/Concurrency/DistributedList.php @@ -16,7 +16,6 @@ * * The list of items is serialized and stored in an Option. Items are converted to string * before being persisted, so it is not expected to unserialize objects. - * TODO: light integration tests */ class DistributedList { @@ -88,7 +87,9 @@ public function add($item) } /** - * Removes one or more items by value from the list in the optiontable. + * Removes one or more items by value from the list in the option table. + * + * Does not preserve array keys. * * @param string|array $items */ @@ -109,6 +110,29 @@ public function remove($items) unset($allItems[$existingIndex]); } - $this->setAll($allItems); + $this->setAll(array_values($allItems)); + } + + /** + * Removes one or more items by index from the list in the option table. + * + * Does not preserve array keys. + * + * @param int[]|int $indices + */ + public function removeByIndex($indices) + { + if (!is_array($indices)) { + $indices = array($indices); + } + + $indices = array_unique($indices); + + $allItems = $this->getAll(); + foreach ($indices as $index) { + unset($allItems[$index]); + } + + $this->setAll(array_values($allItems)); } } \ No newline at end of file diff --git a/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php b/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php new file mode 100644 index 00000000000..0facc27a74c --- /dev/null +++ b/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php @@ -0,0 +1,153 @@ +distributedList = new DistributedList(self::TEST_OPTION_NAME); + + $this->initOptionValue(); + } + + public function test_getAll_CorrectlyReturnsItemsInOption() + { + $list = $this->distributedList->getAll(); + $this->assertEquals(self::$defaultOptionValues, $list); + } + + public function test_getAll_ReturnsValueInOption_IfOptionCacheHasSeparateValue() + { + // get option so cache is loaded + Option::get(self::TEST_OPTION_NAME); + + // set option value to something else + $newList = array('1', '2', '3'); + $this->initOptionValue($newList); + + // test option is now different + $list = $this->distributedList->getAll(); + $this->assertEquals($newList, $list); + } + + public function test_setAll_CorrectlySetsNormalListInOption() + { + $newList = array('1', '2', '3'); + $this->distributedList->setAll($newList); + + $optionValue = $this->getOptionValueForList(); + $this->assertEquals(serialize($newList), $optionValue); + + $list = $this->distributedList->getAll(); + $this->assertEquals($newList, $list); + } + + public function test_setAll_CorrectlyConvertsItemsToString_BeforePersistingToOption() + { + $newList = array('1', Date::factory('2015-02-03'), 4.5); + $this->distributedList->setAll($newList); + + $optionValue = $this->getOptionValueForList(); + $expectedOptionList = array('1', '2015-02-03', '4.5'); + $this->assertEquals(serialize($expectedOptionList), $optionValue); + + $list = $this->distributedList->getAll(); + $this->assertEquals($expectedOptionList, $list); + } + + public function test_add_AddsOneItemToList_InOptionTable_IfItemIsNotArray() + { + $this->distributedList->add('val5'); + + $expectedOptionList = array('val1', 'val2', 'val3', 'val4', 'val5'); + $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList()); + } + + public function test_add_AddsMultipleItemsToList_InOptionTable_IfItemsIsArray() + { + $this->distributedList->add(array('val5', Date::factory('2015-03-04'))); + + $expectedOptionList = array('val1', 'val2', 'val3', 'val4', 'val5', '2015-03-04'); + $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList()); + } + + public function test_remove_RemovesSingleItemByValue_InOptionTable_IfItemIsNotArray() + { + $this->distributedList->remove('val2'); + + $expectedOptionList = array('val1', 'val3', 'val4'); + $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList()); + } + + public function test_remove_RemovesMultipleItemsByValue_InOptionTable_IfItemIsArray() + { + $this->distributedList->remove(array('val2', 'val4')); + + $expectedOptionList = array('val1', 'val3'); + $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList()); + } + + public function test_removeByIndex_RemovesSingleItemByIndex_InOptionTable_IfArgIsIndex() + { + $this->distributedList->removeByIndex(2); + + $expectedOptionList = array('val1', 'val2', 'val4'); + $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList()); + } + + public function test_removeByIndex_RemovesMultipleItemsByIndex_InOptionTable_IfArgIsArray() + { + $this->distributedList->removeByIndex(array(1, 3)); + + $expectedOptionList = array('val1', 'val3'); + $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList()); + } + + private function initOptionValue($data = false) + { + $data = $data ?: self::$defaultOptionValues; + + $optionTable = Common::prefixTable('option'); + Db::query("INSERT INTO `$optionTable` (option_name, option_value, autoload) VALUES (?, ?, ?) + ON DUPLICATE KEY UPDATE option_value = ?", + array(self::TEST_OPTION_NAME, serialize($data), 0, serialize($data))); + } + + private function getOptionValueForList() + { + $optionTable = Common::prefixTable('option'); + return Db::fetchOne("SELECT option_value FROM `$optionTable` WHERE option_name = ?", array(self::TEST_OPTION_NAME)); + } +} \ No newline at end of file From a73225f9c61e3b9cfa6d003daec082d167f2bf4d Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 19:26:34 -0800 Subject: [PATCH 19/31] Do not add command in core/Console.php if command already exists so tests can inject mock commands. --- core/Console.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/Console.php b/core/Console.php index de51756f39c..d36d7b91a4e 100644 --- a/core/Console.php +++ b/core/Console.php @@ -12,6 +12,7 @@ use Piwik\Plugin\Manager as PluginManager; use Symfony\Bridge\Monolog\Handler\ConsoleHandler; use Symfony\Component\Console\Application; +use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; @@ -63,10 +64,14 @@ private function addCommandIfExists($command) { if (!class_exists($command)) { Log::warning(sprintf('Cannot add command %s, class does not exist', $command)); - } elseif (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) { + } else if (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) { Log::warning(sprintf('Cannot add command %s, class does not extend Piwik\Plugin\ConsoleCommand', $command)); } else { - $this->add(new $command); + /** @var Command $commandInstance */ + $commandInstance = new $command; + if (!$this->has($commandInstance->getName())) { + $this->add($commandInstance); + } } } From ca1a5c3c365f3b1e7c3529e0568a9cca60ec754c Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 19:27:30 -0800 Subject: [PATCH 20/31] Refs #7181, adding tests for core:purge-old-archive-data command and make asserts in RawArchiveDataWithTempAndInvalidated more generic --- .../Commands/PurgeOldArchiveData.php | 17 +++++++++-- .../tests/Integration/TasksTest.php | 4 +-- .../RawArchiveDataWithTempAndInvalidated.php | 28 +++++++++++-------- .../TestCase/ConsoleCommandTestCase.php | 18 ++++++++---- .../Integration/Archive/PurgerTest.php | 17 ++++++----- 5 files changed, 53 insertions(+), 31 deletions(-) diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php index 29b1633927d..c8a358e721b 100644 --- a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php +++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php @@ -23,7 +23,6 @@ /** * Command that allows users to force purge old or invalid archive data. In the event of a failure * in the archive purging scheduled task, this command can be used to manually delete old/invalid archives. - * TODO: command tests */ class PurgeOldArchiveData extends ConsoleCommand { @@ -34,6 +33,13 @@ class PurgeOldArchiveData extends ConsoleCommand */ private $archivePurger; + /** + * For tests. + * + * @var Date + */ + public static $todayOverride = null; + public function __construct(Purger $archivePurger = null) { parent::__construct(); @@ -47,7 +53,7 @@ protected function configure() $this->setDescription('Purges old and invalid archive data from archive tables.'); $this->addArgument("dates", InputArgument::IS_ARRAY | InputArgument::OPTIONAL, "The months of the archive tables to purge data from. By default, only deletes from the current month. Use '" . self::ALL_DATES_STRING. "' for all dates.", - array(Date::today()->toString())); + array(self::getToday()->toString())); $this->addOption('exclude-outdated', null, InputOption::VALUE_NONE, "Do not purge outdated archive data."); $this->addOption('exclude-invalidated', null, InputOption::VALUE_NONE, "Do not purge invalidated archive data."); $this->addOption('skip-optimize-tables', null, InputOption::VALUE_NONE, "Do not run OPTIMIZE TABLES query on affected archive tables."); @@ -153,4 +159,9 @@ private function optimizeArchiveTables(OutputInterface $output, $dates) }); } } -} + + private static function getToday() + { + return self::$todayOverride ?: Date::today(); + } +} \ No newline at end of file diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php index aacc20d7f7b..ffb551a464f 100644 --- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php +++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php @@ -60,8 +60,8 @@ public function test_purgeInvalidatedArchives_PurgesCorrectInvalidatedArchives_A $this->tasks->purgeInvalidatedArchives(); - self::$fixture->assertFebruaryInvalidatedArchivesPurged(); - self::$fixture->assertJanuaryInvalidatedArchivesNotPurged(); + self::$fixture->assertInvalidatedArchivesPurged($this->february); + self::$fixture->assertInvalidatedArchivesNotPurged($this->january); // assert invalidated reports distributed list has changed $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList(); diff --git a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php index 080aa3c0431..aa05e9d5b5d 100644 --- a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php +++ b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php @@ -314,7 +314,7 @@ private function setDateMonthAndYear($dateString, Date $archiveDate) return $archiveDate->toString('Y-m') . '-' . Date::factory($dateString)->toString('d'); } - public function assertFebruaryTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled) + public function assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled, Date $date) { if ($isBrowserTriggeredArchivingEnabled) { $expectedPurgedArchives = array(1,2,3,4,6,7); // only archives from 2 hours before "now" are purged @@ -322,25 +322,31 @@ public function assertFebruaryTemporaryArchivesPurged($isBrowserTriggeredArchivi $expectedPurgedArchives = array(1,2,3,4,7); // only archives before start of "yesterday" are purged } - $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $date); } - public function assertFebruaryCustomRangesPurged() + public function assertCustomRangesPurged(Date $date) { $expectedPurgedArchives = array(8,9,10); - $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $date); } - public function assertJanuaryTemporaryArchivesNotPurged() + public function assertTemporaryArchivesNotPurged(Date $date) { $expectedPresentArchives = array(1,2,3,4,5,6,7); - $this->assertArchivesExist($expectedPresentArchives, $this->january); + $this->assertArchivesExist($expectedPresentArchives, $date); } - public function assertJanuaryInvalidatedArchivesNotPurged() + public function assertInvalidatedArchivesNotPurged(Date $date) { $expectedPresentArchives = array(11, 12, 13, 14); - $this->assertArchivesExist($expectedPresentArchives, $this->january); + $this->assertArchivesExist($expectedPresentArchives, $date); + } + + public function assertCustomRangesNotPurged(Date $date) + { + $expectedPresentArchives = array(8, 9, 10); + $this->assertArchivesExist($expectedPresentArchives, $date); } public function assertArchivesDoNotExist($expectedPurgedArchiveIds, $archiveDate) @@ -374,14 +380,14 @@ private function getArchiveRowCountWithId($table, $archiveIds) return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE idarchive IN (".implode(',', $archiveIds).")"); } - public function assertFebruaryInvalidatedArchivesPurged() + public function assertInvalidatedArchivesPurged(Date $date) { // check invalidated archives for all sites are purged $expectedPurgedArchives = array(11, 12, 13, 14); - $this->assertArchivesDoNotExist($expectedPurgedArchives, $this->february); + $this->assertArchivesDoNotExist($expectedPurgedArchives, $date); // check archive 15 is not purged since it doesn't have newer DONE_OK/DONE_TEMPORARY archive $expectedExistingArchives = array(15); - $this->assertArchivesExist($expectedExistingArchives, $this->february); + $this->assertArchivesExist($expectedExistingArchives, $date); } } \ No newline at end of file diff --git a/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php b/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php index f88a62b445c..3a8d33c7a54 100644 --- a/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php +++ b/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php @@ -8,8 +8,8 @@ namespace Piwik\Tests\Framework\TestCase; -use Piwik\Config; use Piwik\Console; +use Symfony\Component\Console\Application; use Symfony\Component\Console\Tester\ApplicationTester; /** @@ -37,18 +37,24 @@ */ class ConsoleCommandTestCase extends SystemTestCase { + /** + * @var ApplicationTester + */ protected $applicationTester = null; + /** + * @var Console + */ + protected $application; + public function setUp() { parent::setUp(); - $application = new Console(); - $application->setAutoExit(false); - - $this->applicationTester = new ApplicationTester($application); + $this->application = new Console(); + $this->application->setAutoExit(false); - Config::unsetInstance(); + $this->applicationTester = new ApplicationTester($this->application); } protected function getCommandDisplayOutputErrorMessage() diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index 2f27127e718..d97d3489544 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -9,7 +9,6 @@ use Piwik\Archive\Purger; use Piwik\Config; -use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; use Piwik\Tests\Fixtures\RawArchiveDataWithTempAndInvalidated; @@ -65,10 +64,10 @@ public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRan $this->archivePurger->purgeOutdatedArchives($this->february); - self::$fixture->assertFebruaryTemporaryArchivesPurged($browserTriggeringEnabled = true); - self::$fixture->assertFebruaryCustomRangesPurged(); + self::$fixture->assertTemporaryArchivesPurged($browserTriggeringEnabled = true, $this->february); + self::$fixture->assertCustomRangesPurged($this->february); - self::$fixture->assertJanuaryTemporaryArchivesNotPurged(); + self::$fixture->assertTemporaryArchivesNotPurged($this->january); } public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRangeArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringDisabled() @@ -77,18 +76,18 @@ public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRan $this->archivePurger->purgeOutdatedArchives($this->february); - self::$fixture->assertFebruaryTemporaryArchivesPurged($browserTriggeringEnabled = false); - self::$fixture->assertFebruaryCustomRangesPurged(); + self::$fixture->assertTemporaryArchivesPurged($browserTriggeringEnabled = false, $this->february); + self::$fixture->assertCustomRangesPurged($this->february); - self::$fixture->assertJanuaryTemporaryArchivesNotPurged(); + self::$fixture->assertTemporaryArchivesNotPurged($this->january); } public function test_purgeInvalidatedArchivesFrom_PurgesAllInvalidatedArchives_AndMarksDatesAndSitesAsInvalidated() { $this->archivePurger->purgeInvalidatedArchivesFrom($this->february); - self::$fixture->assertFebruaryInvalidatedArchivesPurged(); - self::$fixture->assertJanuaryInvalidatedArchivesNotPurged(); + self::$fixture->assertInvalidatedArchivesPurged($this->february); + self::$fixture->assertInvalidatedArchivesNotPurged($this->january); } private function configureCustomRangePurging() From 746b599f03d93ad82a1a270f610f1cf15787a949 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 19:47:08 -0800 Subject: [PATCH 21/31] Refs #7811, do not purge custom ranges in same method as outdated archive purging. Move to a second method in Archive\Purger. --- core/Archive/Purger.php | 4 +--- .../Commands/PurgeOldArchiveData.php | 13 +++++++++++++ plugins/CoreAdminHome/Tasks.php | 5 ++++- .../RawArchiveDataWithTempAndInvalidated.php | 7 +++++-- tests/PHPUnit/Integration/Archive/PurgerTest.php | 16 ++++++++++++---- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/core/Archive/Purger.php b/core/Archive/Purger.php index 3d91170a37f..031decc36d5 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/Purger.php @@ -117,8 +117,6 @@ public function purgeOutdatedArchives(Date $dateStart) $this->deleteArchiveIds($dateStart, $idArchivesToDelete); } - $this->deleteArchivesWithPeriodRange($dateStart); // TODO: this is unrelated to outdated archive purging, should be in its own method - Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]", $purgeArchivesOlderThan, $dateStart->toString("Y-m"), @@ -146,7 +144,7 @@ protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan) * * @param $date Date */ - protected function deleteArchivesWithPeriodRange(Date $date) + public function purgeArchivesWithPeriodRange(Date $date) { $numericTable = ArchiveTableCreator::getNumericTable($date); $blobTable = ArchiveTableCreator::getBlobTable($date); diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php index c8a358e721b..8c88a7e3120 100644 --- a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php +++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php @@ -56,6 +56,7 @@ protected function configure() array(self::getToday()->toString())); $this->addOption('exclude-outdated', null, InputOption::VALUE_NONE, "Do not purge outdated archive data."); $this->addOption('exclude-invalidated', null, InputOption::VALUE_NONE, "Do not purge invalidated archive data."); + $this->addOption('exclude-ranges', null, InputOption::VALUE_NONE, "Do not purge custom ranges."); $this->addOption('skip-optimize-tables', null, InputOption::VALUE_NONE, "Do not run OPTIMIZE TABLES query on affected archive tables."); $this->setHelp("By default old and invalidated archives are purged. Custom ranges are also purged with outdated archives.\n\n" . "Note: archive purging is done during scheduled task execution, so under normal circumstances, you should not need to " @@ -93,6 +94,18 @@ protected function execute(InputInterface $input, OutputInterface $output) } } + $excludeCustomRanges = $input->getOption('exclude-ranges'); + if ($excludeCustomRanges) { + $output->writeln("Skipping purge custom range archives."); + } else { + foreach ($dates as $date) { + $message = sprintf("Purging custom range archives for %s...", $date->toString('Y_m')); + $this->performTimedPurging($output, $message, function () use ($date, $archivePurger) { + $archivePurger->purgeArchivesWithPeriodRange($date); + }); + } + } + $skipOptimizeTables = $input->getOption('skip-optimize-tables'); if ($skipOptimizeTables) { $output->writeln("Skipping OPTIMIZE TABLES."); diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 07398a83e34..ae28788d1dc 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -58,7 +58,10 @@ public function purgeOutdatedArchives() // Somehow we may have archive tables created with older dates, prevent exception from being thrown if ($year > 1990) { - $this->archivePurger->purgeOutdatedArchives(Date::factory("$year-$month-15")); + $dateObj = Date::factory("$year-$month-15"); + + $this->archivePurger->purgeOutdatedArchives($dateObj); + $this->archivePurger->purgeArchivesWithPeriodRange($dateObj); } } } diff --git a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php index aa05e9d5b5d..a6ef25a89f3 100644 --- a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php +++ b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php @@ -343,9 +343,12 @@ public function assertInvalidatedArchivesNotPurged(Date $date) $this->assertArchivesExist($expectedPresentArchives, $date); } - public function assertCustomRangesNotPurged(Date $date) + public function assertCustomRangesNotPurged(Date $date, $includeTemporary = true) { - $expectedPresentArchives = array(8, 9, 10); + $expectedPresentArchives = array(8, 9); + if ($includeTemporary) { + $expectedPresentArchives[] = 10; + } $this->assertArchivesExist($expectedPresentArchives, $date); } diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index d97d3489544..2b3330ac324 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -58,27 +58,27 @@ public function setUp() $this->configureCustomRangePurging(); } - public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRangeArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringEnabled() + public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringEnabled() { $this->enableBrowserTriggeredArchiving(); $this->archivePurger->purgeOutdatedArchives($this->february); self::$fixture->assertTemporaryArchivesPurged($browserTriggeringEnabled = true, $this->february); - self::$fixture->assertCustomRangesPurged($this->february); + self::$fixture->assertCustomRangesNotPurged($this->february, $includeTemporary = false); self::$fixture->assertTemporaryArchivesNotPurged($this->january); } - public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_AndRangeArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringDisabled() + public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringDisabled() { $this->disableBrowserTriggeredArchiving(); $this->archivePurger->purgeOutdatedArchives($this->february); self::$fixture->assertTemporaryArchivesPurged($browserTriggeringEnabled = false, $this->february); - self::$fixture->assertCustomRangesPurged($this->february); + self::$fixture->assertCustomRangesNotPurged($this->february); self::$fixture->assertTemporaryArchivesNotPurged($this->january); } @@ -90,6 +90,14 @@ public function test_purgeInvalidatedArchivesFrom_PurgesAllInvalidatedArchives_A self::$fixture->assertInvalidatedArchivesNotPurged($this->january); } + public function test_purgeArchivesWithPeriodRange_PurgesAllRangeArchives() + { + $this->archivePurger->purgeArchivesWithPeriodRange($this->february); + + self::$fixture->assertCustomRangesPurged($this->february); + self::$fixture->assertCustomRangesNotPurged($this->january); + } + private function configureCustomRangePurging() { Config::getInstance()->General['purge_date_range_archives_after_X_days'] = 3; From ca783efd8f63f6efbd470bcb76dc7176e8bae9f3 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 19:47:55 -0800 Subject: [PATCH 22/31] Refs #7811, forgot to add new command test. --- .../Commands/PurgeOldArchiveDataTest.php | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php diff --git a/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php b/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php new file mode 100644 index 00000000000..1930b414da6 --- /dev/null +++ b/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php @@ -0,0 +1,148 @@ +setTodayDate(Date::factory('2015-02-27')); + $archivePurger->setYesterdayDate(Date::factory('2015-02-26')); + $archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp()); + + $this->application = new Console(); + $this->application->setAutoExit(false); + $this->application->add(new PurgeOldArchiveData($archivePurger)); + + $this->applicationTester = new ApplicationTester($this->application); + } + + public function tearDown() + { + PurgeOldArchiveData::$todayOverride = null; + + parent::tearDown(); + } + + public function test_ExecutingCommandWithAllDates_PurgesAllExistingArchiveTables() + { + $result = $this->applicationTester->run(array( + 'command' => 'core:purge-old-archive-data', + 'dates' => array('all'), + '-vvv' => true + )); + + $this->assertEquals(0, $result, $this->getCommandDisplayOutputErrorMessage()); + + self::$fixture->assertInvalidatedArchivesPurged(self::$fixture->february); + self::$fixture->assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled = true, self::$fixture->february); + self::$fixture->assertCustomRangesPurged(self::$fixture->february); + + self::$fixture->assertInvalidatedArchivesPurged(self::$fixture->january); + self::$fixture->assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled = true, self::$fixture->january); + self::$fixture->assertCustomRangesPurged(self::$fixture->january); + } + + public function test_ExecutingCommandWithNoDate_PurgesArchiveTableForToday() + { + $result = $this->applicationTester->run(array( + 'command' => 'core:purge-old-archive-data', + '-vvv' => true + )); + + $this->assertEquals(0, $result, $this->getCommandDisplayOutputErrorMessage()); + + self::$fixture->assertInvalidatedArchivesPurged(self::$fixture->february); + self::$fixture->assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled = true, self::$fixture->february); + self::$fixture->assertCustomRangesPurged(self::$fixture->february); + + self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january); + self::$fixture->assertTemporaryArchivesNotPurged(self::$fixture->january); + self::$fixture->assertCustomRangesNotPurged(self::$fixture->january); + } + + public function test_ExecutingCommandWithSpecificDate_PurgesArchiveTableForDate() + { + $result = $this->applicationTester->run(array( + 'command' => 'core:purge-old-archive-data', + 'dates' => array('2015-01-14'), + '-vvv' => true + )); + + $this->assertEquals(0, $result, $this->getCommandDisplayOutputErrorMessage()); + + self::$fixture->assertInvalidatedArchivesPurged(self::$fixture->january); + self::$fixture->assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled = true, self::$fixture->january); + self::$fixture->assertCustomRangesPurged(self::$fixture->january); + + self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->february); + self::$fixture->assertTemporaryArchivesNotPurged(self::$fixture->february); + self::$fixture->assertCustomRangesNotPurged(self::$fixture->february); + } + + public function test_ExecutingCommandWithExcludeOptions_SkipsAppropriatePurging() + { + $result = $this->applicationTester->run(array( + 'command' => 'core:purge-old-archive-data', + 'dates' => array('2015-01-14'), + '--exclude-outdated' => true, + '--exclude-invalidated' => true, + '--exclude-ranges' => true, + '--skip-optimize-tables' => true, + '-vvv' => true + )); + + $this->assertEquals(0, $result, $this->getCommandDisplayOutputErrorMessage()); + + self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january); + self::$fixture->assertTemporaryArchivesNotPurged(self::$fixture->january); + self::$fixture->assertCustomRangesNotPurged(self::$fixture->january); + + $this->assertContains("Skipping purge outdated archive data.", $this->applicationTester->getDisplay()); + $this->assertContains("Skipping purge invalidated archive data.", $this->applicationTester->getDisplay()); + $this->assertContains("Skipping OPTIMIZE TABLES.", $this->applicationTester->getDisplay()); + } + + protected function getCommandDisplayOutputErrorMessage() + { + return "Command did not behave as expected. Command output: " . $this->applicationTester->getDisplay(); + } +} + +PurgeOldArchiveDataTest::$fixture = new RawArchiveDataWithTempAndInvalidated(); \ No newline at end of file From 37c69f41b5edc5db71dba3fcdcd4a744fe386461 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 5 Mar 2015 22:16:01 -0800 Subject: [PATCH 23/31] Refs #7181, clarify code + comment that skips archiving in scheduled task. --- plugins/CoreAdminHome/Tasks.php | 16 ++++++++++------ tests/PHPUnit/Integration/Archive/PurgerTest.php | 4 ---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index ae28788d1dc..477f3863442 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -15,6 +15,7 @@ use Piwik\Db; use Piwik\Log; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; +use Piwik\SettingsServer; class Tasks extends \Piwik\Plugin\Tasks { @@ -42,12 +43,15 @@ public function schedule() public function purgeOutdatedArchives() { - // TODO: is this correct? wouldn't segment archives create DONE archives, not DONE_TEMPORARY? should try to replicate in tests - // we only delete archives if we are able to process them, otherwise, the browser might process reports - // when &segment= is specified (or custom date range) and would below, delete temporary archives that the - // browser is not able to process until next cron run (which could be more than 1 hour away) - if (!Rules::isRequestAuthorizedToArchive()) { - Log::info("Purging temporary archives: skipped (request not allowed to initiate archiving)"); + // we should only purge outdated & custom range archives if we know cron archiving has just run, + // or if browser triggered archiving is enabled. if cron archiving has run, then we know the latest + // archives are in the database, and we can remove temporary ones. if browser triggered archiving is + // enabled, then we know any archives that are wrongly purged, can be re-archived on demand. + // this prevents some situations where "no data" is displayed for reports that should have data. + if (!Rules::isBrowserTriggerEnabled() + && !SettingsServer::isArchivePhpTriggered() + ) { + Log::info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)"); return false; } diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index 2b3330ac324..4111a28ca95 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -19,10 +19,6 @@ */ class PurgerTest extends IntegrationTestCase { - /* TODO: things to test - * - that archive purging is done daily (when called through cron archiving and through API request for scheduled task running) [SYSTEM TESTS] - */ - /** * @var RawArchiveDataWithTempAndInvalidated */ From db686a6f900e2d64467bee4a3987ebd5052ac794 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Fri, 6 Mar 2015 00:31:45 -0800 Subject: [PATCH 24/31] Adding log statement + TODO to CoreAdminHome/Tasks.php. --- plugins/CoreAdminHome/Tasks.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 477f3863442..22a2fbb1a6f 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -56,6 +56,10 @@ public function purgeOutdatedArchives() } $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled(); + + Log::info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables))); + + // TODO: won't this execute purges twice, since this includes blobs + numeric tables, but purging should be called w/ just the date? foreach ($archiveTables as $table) { $date = ArchiveTableCreator::getDateFromTableName($table); list($year, $month) = explode('_', $date); From 2c80f5bc4bf655b95c0517af9d8e88d0a8195ae7 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Mon, 9 Mar 2015 22:48:51 -0700 Subject: [PATCH 25/31] Add comment in core\Console.php regarding not adding a command if it already exists in the Piwik Console class. --- core/Console.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/Console.php b/core/Console.php index d36d7b91a4e..a598edffacd 100644 --- a/core/Console.php +++ b/core/Console.php @@ -69,6 +69,8 @@ private function addCommandIfExists($command) } else { /** @var Command $commandInstance */ $commandInstance = new $command; + + // do not add the command if it already exists; this way we can add the command ourselves in tests if (!$this->has($commandInstance->getName())) { $this->add($commandInstance); } From 98510ab6ee3a210f9b9ef85b5f74d88c2c3a69e2 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Mon, 9 Mar 2015 22:52:44 -0700 Subject: [PATCH 26/31] Rename Archive\Invalidator to ArchiveInvalidator and Archive\Purger for ArchivePurger for clarity. --- core/Archive.php | 4 ++-- .../{Invalidator.php => ArchiveInvalidator.php} | 2 +- core/Archive/{Purger.php => ArchivePurger.php} | 5 ++--- core/CronArchive.php | 4 ++-- core/Tracker/Visit.php | 4 ++-- plugins/CoreAdminHome/API.php | 4 ++-- .../CoreAdminHome/Commands/FixDuplicateLogActions.php | 10 +++++----- plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php | 8 ++++---- plugins/CoreAdminHome/Tasks.php | 8 ++++---- .../Integration/Commands/PurgeOldArchiveDataTest.php | 4 ++-- plugins/CoreAdminHome/tests/Integration/TasksTest.php | 4 ++-- plugins/SitesManager/SitesManager.php | 4 ++-- .../tests/Integration/SitesManagerTest.php | 4 ++-- tests/PHPUnit/Integration/Archive/PurgerTest.php | 6 +++--- tests/PHPUnit/Integration/CronArchiveTest.php | 4 ++-- .../Integration/DataAccess/ArchiveInvalidatorTest.php | 8 ++++---- tests/PHPUnit/Integration/Tracker/VisitTest.php | 4 ++-- ...VisitorsTwoWebsitesDifferentDaysConversionsTest.php | 4 ++-- ...sWithCustomVariablesSegmentMatchVisitorTypeTest.php | 2 +- 19 files changed, 46 insertions(+), 47 deletions(-) rename core/Archive/{Invalidator.php => ArchiveInvalidator.php} (99%) rename core/Archive/{Purger.php => ArchivePurger.php} (98%) diff --git a/core/Archive.php b/core/Archive.php index d8032ce7e6c..c95a37ddd35 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -10,7 +10,7 @@ use Piwik\Archive\Parameters; use Piwik\ArchiveProcessor\Rules; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\DataAccess\ArchiveSelector; use Piwik\Period\Factory as PeriodFactory; @@ -517,7 +517,7 @@ private function invalidatedReportsIfNeeded() return; // all requested site ids were already handled } - $invalidator = new Invalidator(); + $invalidator = new ArchiveInvalidator(); $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); foreach ($sitesPerDays as $date => $siteIds) { diff --git a/core/Archive/Invalidator.php b/core/Archive/ArchiveInvalidator.php similarity index 99% rename from core/Archive/Invalidator.php rename to core/Archive/ArchiveInvalidator.php index bdb4dd9bf61..e9ce777cc49 100644 --- a/core/Archive/Invalidator.php +++ b/core/Archive/ArchiveInvalidator.php @@ -45,7 +45,7 @@ * Later, during Piwik's normal execution, the list will be read and every archive it references will * be invalidated. */ -class Invalidator +class ArchiveInvalidator { private $warningDates = array(); private $processedDates = array(); diff --git a/core/Archive/Purger.php b/core/Archive/ArchivePurger.php similarity index 98% rename from core/Archive/Purger.php rename to core/Archive/ArchivePurger.php index 031decc36d5..e41ce375c55 100644 --- a/core/Archive/Purger.php +++ b/core/Archive/ArchivePurger.php @@ -16,7 +16,6 @@ use Piwik\Db; use Piwik\Log; use Piwik\Piwik; -use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; /** * Service that purges temporary, error-ed, invalid and custom range archives from archive tables. @@ -27,9 +26,9 @@ * Error-ed archives are purged w/o constraint. * * Invalid archives are purged if a new, valid, archive exists w/ the same site, date, period combination. - * Archives are marked as invalid via Piwik\Archive\Invalidator. + * Archives are marked as invalid via Piwik\Archive\ArchiveInvalidator. */ -class Purger +class ArchivePurger { /** * @var Model diff --git a/core/CronArchive.php b/core/CronArchive.php index 12991dab72e..0ff2ee82422 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -12,7 +12,7 @@ use Piwik\ArchiveProcessor\Rules; use Piwik\CronArchive\FixedSiteIds; use Piwik\CronArchive\SharedSiteIds; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Metrics\Formatter; use Piwik\Period\Factory as PeriodFactory; @@ -973,7 +973,7 @@ private function getApiToInvalidateArchivedReport() public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain() { - $invalidator = new Invalidator(); + $invalidator = new ArchiveInvalidator(); $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); foreach ($sitesPerDays as $date => $siteIds) { diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php index 3a149cfbd04..f672cd96da2 100644 --- a/core/Tracker/Visit.php +++ b/core/Tracker/Visit.php @@ -11,7 +11,7 @@ use Piwik\Common; use Piwik\Config; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Date; use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Network\IPUtils; @@ -682,7 +682,7 @@ private function markArchivedReportsAsInvalidIfArchiveAlreadyFinished() $date = Date::factory((int) $time, $timezone); if (!$date->isToday()) { // we don't have to handle in case date is in future as it is not allowed by tracker - $invalidReport = new Invalidator(); + $invalidReport = new ArchiveInvalidator(); $invalidReport->rememberToInvalidateArchivedReportsLater($idSite, $date); } } diff --git a/plugins/CoreAdminHome/API.php b/plugins/CoreAdminHome/API.php index d3041a170dc..6329fc89936 100644 --- a/plugins/CoreAdminHome/API.php +++ b/plugins/CoreAdminHome/API.php @@ -10,7 +10,7 @@ use Exception; use Piwik\Container\StaticContainer; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Db; use Piwik\Piwik; use Piwik\Scheduler\Scheduler; @@ -69,7 +69,7 @@ public function invalidateArchivedReports($idSites, $dates, $period = false) Piwik::checkUserHasAdminAccess($idSites); - $invalidator = new Invalidator(); + $invalidator = new ArchiveInvalidator(); $output = $invalidator->markArchivesAsInvalidated($idSites, $dates, $period); Site::clearCache(); diff --git a/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php b/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php index b26b866e27a..be9c74a1527 100644 --- a/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php +++ b/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php @@ -11,7 +11,7 @@ use Piwik\Common; use Piwik\Container\StaticContainer; use Piwik\DataAccess\Actions; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Plugin\ConsoleCommand; use Piwik\Plugins\CoreAdminHome\Model\DuplicateActionRemover; use Piwik\Timer; @@ -39,7 +39,7 @@ class FixDuplicateLogActions extends ConsoleCommand /** * Used to invalidate archives. Only used if $shouldInvalidateArchives is true. * - * @var Invalidator + * @var ArchiveInvalidator */ private $archiveInvalidator; @@ -65,17 +65,17 @@ class FixDuplicateLogActions extends ConsoleCommand /** * Constructor. * - * @param Invalidator $invalidator + * @param ArchiveInvalidator $invalidator * @param DuplicateActionRemover $duplicateActionRemover * @param Actions $actionsAccess * @param LoggerInterface $logger */ - public function __construct(Invalidator $invalidator = null, DuplicateActionRemover $duplicateActionRemover = null, + public function __construct(ArchiveInvalidator $invalidator = null, DuplicateActionRemover $duplicateActionRemover = null, Actions $actionsAccess = null, LoggerInterface $logger = null) { parent::__construct(); - $this->archiveInvalidator = $invalidator ?: new Invalidator(); + $this->archiveInvalidator = $invalidator ?: new ArchiveInvalidator(); $this->duplicateActionRemover = $duplicateActionRemover ?: new DuplicateActionRemover(); $this->actionsAccess = $actionsAccess ?: new Actions(); $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface'); diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php index 8c88a7e3120..dcea37f0219 100644 --- a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php +++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php @@ -9,7 +9,7 @@ namespace Piwik\Plugins\CoreAdminHome\Commands; use Piwik\Archive; -use Piwik\Archive\Purger; +use Piwik\Archive\ArchivePurger; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; @@ -29,7 +29,7 @@ class PurgeOldArchiveData extends ConsoleCommand const ALL_DATES_STRING = 'all'; /** - * @var Purger + * @var ArchivePurger */ private $archivePurger; @@ -40,11 +40,11 @@ class PurgeOldArchiveData extends ConsoleCommand */ public static $todayOverride = null; - public function __construct(Purger $archivePurger = null) + public function __construct(ArchivePurger $archivePurger = null) { parent::__construct(); - $this->archivePurger = $archivePurger ?: new Purger(); + $this->archivePurger = $archivePurger ?: new ArchivePurger(); } protected function configure() diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 22a2fbb1a6f..b6856f5ef8f 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -9,7 +9,7 @@ namespace Piwik\Plugins\CoreAdminHome; use Piwik\ArchiveProcessor\Rules; -use Piwik\Archive\Purger; +use Piwik\Archive\ArchivePurger; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; @@ -20,13 +20,13 @@ class Tasks extends \Piwik\Plugin\Tasks { /** - * @var Purger + * @var ArchivePurger */ private $archivePurger; - public function __construct(Purger $archivePurger = null) + public function __construct(ArchivePurger $archivePurger = null) { - $this->archivePurger = $archivePurger ?: new Purger(); + $this->archivePurger = $archivePurger ?: new ArchivePurger(); } public function schedule() diff --git a/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php b/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php index 1930b414da6..5728c6de745 100644 --- a/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php +++ b/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php @@ -7,7 +7,7 @@ */ namespace Piwik\Plugins\CoreAdminHome\tests\Integration\Commands; -use Piwik\Archive\Purger; +use Piwik\Archive\ArchivePurger; use Piwik\Console; use Piwik\Date; use Piwik\Plugins\CoreAdminHome\Commands\PurgeOldArchiveData; @@ -41,7 +41,7 @@ public function setUp() PurgeOldArchiveData::$todayOverride = Date::factory('2015-02-27'); - $archivePurger = new Purger(); + $archivePurger = new ArchivePurger(); $archivePurger->setTodayDate(Date::factory('2015-02-27')); $archivePurger->setYesterdayDate(Date::factory('2015-02-26')); $archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp()); diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php index ffb551a464f..da3800e9c19 100644 --- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php +++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php @@ -7,7 +7,7 @@ */ namespace Piwik\Plugins\CoreAdminHome\tests\Integration; -use Piwik\Archive\Purger; +use Piwik\Archive\ArchivePurger; use Piwik\Date; use Piwik\Plugins\CoreAdminHome\Tasks; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; @@ -46,7 +46,7 @@ public function setUp() $this->january = Date::factory('2015-01-01'); $this->february = Date::factory('2015-02-01'); - $archivePurger = new Purger(); + $archivePurger = new ArchivePurger(); $archivePurger->setTodayDate(Date::factory('2015-02-27')); $archivePurger->setYesterdayDate(Date::factory('2015-02-26')); $archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp()); diff --git a/plugins/SitesManager/SitesManager.php b/plugins/SitesManager/SitesManager.php index d88c2e7c417..ca47a505404 100644 --- a/plugins/SitesManager/SitesManager.php +++ b/plugins/SitesManager/SitesManager.php @@ -7,7 +7,7 @@ * */ namespace Piwik\Plugins\SitesManager; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Tracker\Cache; /** @@ -38,7 +38,7 @@ public function onSiteDeleted($idSite) // we do not delete logs here on purpose (you can run these queries on the log_ tables to delete all data) Cache::deleteCacheWebsiteAttributes($idSite); - $archiveInvalidator = new Invalidator(); + $archiveInvalidator = new ArchiveInvalidator(); $archiveInvalidator->forgetRememberedArchivedReportsToInvalidateForSite($idSite); } diff --git a/plugins/SitesManager/tests/Integration/SitesManagerTest.php b/plugins/SitesManager/tests/Integration/SitesManagerTest.php index 9fc69fdfde1..1ba60ce972f 100644 --- a/plugins/SitesManager/tests/Integration/SitesManagerTest.php +++ b/plugins/SitesManager/tests/Integration/SitesManagerTest.php @@ -10,7 +10,7 @@ use Piwik\Access; use Piwik\Cache; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Date; use Piwik\Plugins\SitesManager\SitesManager; use Piwik\Tests\Framework\Fixture; @@ -56,7 +56,7 @@ public function test_onSiteDeleted_shouldClearSiteCache() public function test_onSiteDeleted_shouldRemoveRememberedArchiveReports() { - $archive = new Invalidator(); + $archive = new ArchiveInvalidator(); $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-05')); $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-06')); $archive->rememberToInvalidateArchivedReportsLater(4949, Date::factory('2014-04-05')); diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index 4111a28ca95..ea0cec316ef 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -7,7 +7,7 @@ */ namespace Piwik\Tests\Integration\Archive; -use Piwik\Archive\Purger; +use Piwik\Archive\ArchivePurger; use Piwik\Config; use Piwik\Date; use Piwik\Db; @@ -25,7 +25,7 @@ class PurgerTest extends IntegrationTestCase public static $fixture; /** - * @var Purger + * @var ArchivePurger */ private $archivePurger; @@ -46,7 +46,7 @@ public function setUp() $this->january = self::$fixture->january; $this->february = self::$fixture->february; - $this->archivePurger = new Purger(); + $this->archivePurger = new ArchivePurger(); $this->archivePurger->setTodayDate(Date::factory('2015-02-27')); $this->archivePurger->setYesterdayDate(Date::factory('2015-02-26')); $this->archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp()); diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php index 9397fe511cb..c90e64a8466 100644 --- a/tests/PHPUnit/Integration/CronArchiveTest.php +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -9,7 +9,7 @@ namespace Piwik\Tests\Integration; use Piwik\CronArchive; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Date; use Piwik\Db; use Piwik\Plugins\CoreAdminHome\tests\Framework\Mock\API; @@ -46,7 +46,7 @@ public function setUp() public function test_getColumnNamesFromTable() { - $ar = new Invalidator(); + $ar = new ArchiveInvalidator(); $ar->rememberToInvalidateArchivedReportsLater(1, Date::factory('2014-04-05')); $ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-05')); $ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-06')); diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index 24e178b1537..7847abc202a 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -11,17 +11,17 @@ use Piwik\Date; use Piwik\Option; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; /** * @group Archiver - * @group Invalidator + * @group ArchiveInvalidator * @group DataAccess */ class ArchiveInvalidatorTest extends IntegrationTestCase { /** - * @var Invalidator + * @var ArchiveInvalidator */ private $invalidator; @@ -29,7 +29,7 @@ public function setUp() { parent::setUp(); - $this->invalidator = new Invalidator(); + $this->invalidator = new ArchiveInvalidator(); } public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet() diff --git a/tests/PHPUnit/Integration/Tracker/VisitTest.php b/tests/PHPUnit/Integration/Tracker/VisitTest.php index ed767b1ea0e..919a918516c 100644 --- a/tests/PHPUnit/Integration/Tracker/VisitTest.php +++ b/tests/PHPUnit/Integration/Tracker/VisitTest.php @@ -11,7 +11,7 @@ use Piwik\Access; use Piwik\Cache; use Piwik\CacheId; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Date; use Piwik\Network\IPUtils; use Piwik\Plugin\Manager; @@ -370,7 +370,7 @@ private function assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $visit->handle(); - $archive = new Invalidator(); + $archive = new ArchiveInvalidator(); $remembered = $archive->getRememberedArchivedReportsThatShouldBeInvalidated(); $this->assertSame($expectedRemeberedArchivedReports, $remembered); diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php index 3c1509f0c6a..7e9a576b71c 100755 --- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php +++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php @@ -9,7 +9,7 @@ use Piwik\Archive; use Piwik\Cache; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Option; use Piwik\Plugins\Goals\Archiver; use Piwik\Segment; @@ -172,7 +172,7 @@ public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePer $this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2), $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated')); - $invalidator = new Invalidator(); + $invalidator = new ArchiveInvalidator(); self::$fixture->trackVisits(); diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index 4a74c954bc1..77bb4f8dac1 100755 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -8,7 +8,7 @@ namespace Piwik\Tests\System; use Piwik\Common; -use Piwik\Archive\Invalidator; +use Piwik\Archive\ArchiveInvalidator; use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\Db; use Piwik\Tests\Framework\TestCase\SystemTestCase; From b828d5f6b40e5a182a6e18db04f740fd309af758 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Mon, 9 Mar 2015 22:55:00 -0700 Subject: [PATCH 27/31] Remove some unneeded documentation. --- core/CronArchive/SitesToReprocessDistributedList.php | 3 --- plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php | 3 --- 2 files changed, 6 deletions(-) diff --git a/core/CronArchive/SitesToReprocessDistributedList.php b/core/CronArchive/SitesToReprocessDistributedList.php index 2ec471aec58..44b1eedca66 100644 --- a/core/CronArchive/SitesToReprocessDistributedList.php +++ b/core/CronArchive/SitesToReprocessDistributedList.php @@ -22,9 +22,6 @@ class SitesToReprocessDistributedList extends DistributedList { const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds'; - /** - * Constructor. - */ public function __construct() { parent::__construct(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS); diff --git a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php index a7c9b6eb9ab..6d562866329 100644 --- a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php +++ b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php @@ -27,9 +27,6 @@ class ArchivesToPurgeDistributedList extends DistributedList { const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds'; - /** - * Constructor. - */ public function __construct() { parent::__construct(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE); From 5197b1cd4ddd21e7c2ab4cfe2906dff92ac35734 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Mon, 9 Mar 2015 23:26:02 -0700 Subject: [PATCH 28/31] In purging scheduled task, only purge once per year-month combination found, not once per archive table since dates can appear twice (once for numeric table & once for blob table). --- plugins/CoreAdminHome/Tasks.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index b6856f5ef8f..df2bb2b3b03 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -59,17 +59,24 @@ public function purgeOutdatedArchives() Log::info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables))); - // TODO: won't this execute purges twice, since this includes blobs + numeric tables, but purging should be called w/ just the date? + // keep track of dates we purge for, since getTablesArchivesInstalled() will return numeric & blob + // tables (so dates will appear two times, and we should only purge once per date) + $datesPurged = array(); + foreach ($archiveTables as $table) { $date = ArchiveTableCreator::getDateFromTableName($table); list($year, $month) = explode('_', $date); // Somehow we may have archive tables created with older dates, prevent exception from being thrown - if ($year > 1990) { + if ($year > 1990 + && empty($datesPurged[$date]) + ) { $dateObj = Date::factory("$year-$month-15"); $this->archivePurger->purgeOutdatedArchives($dateObj); $this->archivePurger->purgeArchivesWithPeriodRange($dateObj); + + $datesPurged[$date] = true; } } } From 984d749a0190c5750c510cb9cba97173afffbc21 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 12 Mar 2015 02:28:40 -0700 Subject: [PATCH 29/31] Tweak to core:purge-old-archive-data command help and add command to CHANGELOG.md. --- CHANGELOG.md | 3 +++ plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82a9d15f109..321875df34e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ This is a changelog for Piwik platform developers. All changes for our HTTP API' * `isIpInRange()` * `getHostByAddr()` +### New commands +* There is now a command `core:purge-old-archive-data` that can be used to manually purge temporary, error-ed and invalidated archives from one or more archive tables. + ## Piwik 2.11.0 ### Breaking Changes diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php index dcea37f0219..89696eaa9b3 100644 --- a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php +++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php @@ -50,7 +50,7 @@ public function __construct(ArchivePurger $archivePurger = null) protected function configure() { $this->setName('core:purge-old-archive-data'); - $this->setDescription('Purges old and invalid archive data from archive tables.'); + $this->setDescription('Purges out of date and invalid archive data from archive tables.'); $this->addArgument("dates", InputArgument::IS_ARRAY | InputArgument::OPTIONAL, "The months of the archive tables to purge data from. By default, only deletes from the current month. Use '" . self::ALL_DATES_STRING. "' for all dates.", array(self::getToday()->toString())); From 19325b3bc025a8664e3eb6369a17830768549f46 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Thu, 12 Mar 2015 02:35:58 -0700 Subject: [PATCH 30/31] Add asserts to setUp method of archive purging tests so there is an assurance that the test data is correct. Use Psr logger in purging scheduled task. And use isRequestAuthorizedToArchive() scheduled task in private method that is named well. --- plugins/CoreAdminHome/Tasks.php | 30 ++++++++++++------- .../Commands/PurgeOldArchiveDataTest.php | 4 +++ .../Integration/Archive/PurgerTest.php | 4 +++ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index df2bb2b3b03..700c7d69c27 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -10,6 +10,7 @@ use Piwik\ArchiveProcessor\Rules; use Piwik\Archive\ArchivePurger; +use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; @@ -43,21 +44,16 @@ public function schedule() public function purgeOutdatedArchives() { - // we should only purge outdated & custom range archives if we know cron archiving has just run, - // or if browser triggered archiving is enabled. if cron archiving has run, then we know the latest - // archives are in the database, and we can remove temporary ones. if browser triggered archiving is - // enabled, then we know any archives that are wrongly purged, can be re-archived on demand. - // this prevents some situations where "no data" is displayed for reports that should have data. - if (!Rules::isBrowserTriggerEnabled() - && !SettingsServer::isArchivePhpTriggered() - ) { - Log::info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)"); + $logger = StaticContainer::get('Psr\Log\LoggerInterface'); + + if ($this->willPurgingCausePotentialProblemInUI()) { + $logger->info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)"); return false; } $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled(); - Log::info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables))); + $logger->info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables))); // keep track of dates we purge for, since getTablesArchivesInstalled() will return numeric & blob // tables (so dates will appear two times, and we should only purge once per date) @@ -96,4 +92,18 @@ public function optimizeArchiveTable() $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled(); Db::optimizeTables($archiveTables); } + + /** + * we should only purge outdated & custom range archives if we know cron archiving has just run, + * or if browser triggered archiving is enabled. if cron archiving has run, then we know the latest + * archives are in the database, and we can remove temporary ones. if browser triggered archiving is + * enabled, then we know any archives that are wrongly purged, can be re-archived on demand. + * this prevents some situations where "no data" is displayed for reports that should have data. + * + * @return bool + */ + private function willPurgingCausePotentialProblemInUI() + { + return !Rules::isRequestAuthorizedToArchive(); + } } \ No newline at end of file diff --git a/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php b/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php index 5728c6de745..ecb07aa4382 100644 --- a/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php +++ b/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php @@ -51,6 +51,10 @@ public function setUp() $this->application->add(new PurgeOldArchiveData($archivePurger)); $this->applicationTester = new ApplicationTester($this->application); + + // assert the test data was setup correctly + self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january); + self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->february); } public function tearDown() diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php index ea0cec316ef..a4224734799 100644 --- a/tests/PHPUnit/Integration/Archive/PurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php @@ -52,6 +52,10 @@ public function setUp() $this->archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp()); $this->configureCustomRangePurging(); + + // assert test data was added correctly + self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january); + self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->february); } public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringEnabled() From f19f7fe42fbd129e7b28813c4dd110d9d0e8b558 Mon Sep 17 00:00:00 2001 From: Matthieu Aubry Date: Fri, 13 Mar 2015 15:07:39 +1300 Subject: [PATCH 31/31] add use Piwik\Common; --- plugins/SitesManager/SitesManager.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/SitesManager/SitesManager.php b/plugins/SitesManager/SitesManager.php index 6427dec8757..93e1d8c2e81 100644 --- a/plugins/SitesManager/SitesManager.php +++ b/plugins/SitesManager/SitesManager.php @@ -7,6 +7,8 @@ * */ namespace Piwik\Plugins\SitesManager; + +use Piwik\Common; use Piwik\Archive\ArchiveInvalidator; use Piwik\Tracker\Cache; use Piwik\Tracker\Model as TrackerModel;