From 90695622e41f4913ced772781f7d3dbbbb16af0f Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 2 Feb 2015 01:53:43 +0000 Subject: [PATCH 1/4] do no longer request all idsites by default as it would otherwise load the metadata for each site and if one has 20k sites would request it 20k times! --- core/API/DataTableManipulator.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/API/DataTableManipulator.php b/core/API/DataTableManipulator.php index a1acc6e5b53..862f2db0878 100644 --- a/core/API/DataTableManipulator.php +++ b/core/API/DataTableManipulator.php @@ -124,7 +124,7 @@ protected function loadSubtable($dataTable, $row) } } - $method = $this->getApiMethodForSubtable(); + $method = $this->getApiMethodForSubtable($request); return $this->callApiAndReturnDataTable($this->apiModule, $method, $request); } @@ -144,10 +144,16 @@ protected abstract function manipulateSubtableRequest($request); * @throws Exception * @return string */ - private function getApiMethodForSubtable() + private function getApiMethodForSubtable($request) { if (!$this->apiMethodForSubtable) { - $meta = API::getInstance()->getMetadata('all', $this->apiModule, $this->apiMethod); + if (!empty($request['idSite'])) { + $idSite = $request['idSite']; + } else { + $idSite = 'all'; + } + + $meta = API::getInstance()->getMetadata($idSite, $this->apiModule, $this->apiMethod); if (empty($meta)) { throw new Exception(sprintf( From 478d539a61bc40a73a908aeed65efb0728360dfc Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 2 Feb 2015 01:55:11 +0000 Subject: [PATCH 2/4] cache the returned value of those methods otherwise each of them triggers an event for each label in a datatable which can be many! --- core/Common.php | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/core/Common.php b/core/Common.php index dcadaa2d494..0fe9e341192 100644 --- a/core/Common.php +++ b/core/Common.php @@ -829,11 +829,20 @@ public static function getLanguageToCountryList() */ public static function getSearchEngineUrls() { - require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/SearchEngines.php'; + $cacheId = 'Common.getSearchEngineUrls'; + $cache = Cache::getTransientCache(); + $searchEngines = $cache->fetch($cacheId); - $searchEngines = $GLOBALS['Piwik_SearchEngines']; + if (empty($searchEngines)) { - Piwik::postEvent('Referrer.addSearchEngineUrls', array(&$searchEngines)); + require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/SearchEngines.php'; + + $searchEngines = $GLOBALS['Piwik_SearchEngines']; + + Piwik::postEvent('Referrer.addSearchEngineUrls', array(&$searchEngines)); + + $cache->save($cacheId, $searchEngines); + } return $searchEngines; } @@ -847,13 +856,21 @@ public static function getSearchEngineUrls() */ public static function getSearchEngineNames() { - $searchEngines = self::getSearchEngineUrls(); + $cacheId = 'Common.getSearchEngineNames'; + $cache = Cache::getTransientCache(); + $nameToUrl = $cache->fetch($cacheId); + + if (empty($nameToUrl)) { + + $searchEngines = self::getSearchEngineUrls(); - $nameToUrl = array(); - foreach ($searchEngines as $url => $info) { - if (!isset($nameToUrl[$info[0]])) { - $nameToUrl[$info[0]] = $url; + $nameToUrl = array(); + foreach ($searchEngines as $url => $info) { + if (!isset($nameToUrl[$info[0]])) { + $nameToUrl[$info[0]] = $url; + } } + $cache->save($cacheId, $nameToUrl); } return $nameToUrl; @@ -868,11 +885,20 @@ public static function getSearchEngineNames() */ public static function getSocialUrls() { - require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php'; + $cacheId = 'Common.getSocialUrls'; + $cache = Cache::getTransientCache(); + $socialUrls = $cache->fetch($cacheId); + + if (empty($socialUrls)) { + + require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php'; - $socialUrls = $GLOBALS['Piwik_socialUrl']; + $socialUrls = $GLOBALS['Piwik_socialUrl']; - Piwik::postEvent('Referrer.addSocialUrls', array(&$socialUrls)); + Piwik::postEvent('Referrer.addSocialUrls', array(&$socialUrls)); + + $cache->save($cacheId, $socialUrls); + } return $socialUrls; } From e349e24aa244d7aaaa364cc326cc76285e02b143 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 2 Feb 2015 01:56:55 +0000 Subject: [PATCH 3/4] we do request the available view datatables many times, eg when detecting the footerIcons. Instead cache the result to save many postEvents etc --- core/ViewDataTable/Manager.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/ViewDataTable/Manager.php b/core/ViewDataTable/Manager.php index 5d056506574..aff17743689 100644 --- a/core/ViewDataTable/Manager.php +++ b/core/ViewDataTable/Manager.php @@ -8,6 +8,7 @@ */ namespace Piwik\ViewDataTable; +use Piwik\Cache; use Piwik\Common; use Piwik\Option; use Piwik\Piwik; @@ -63,6 +64,14 @@ public static function getIdsWithInheritance($klass) */ public static function getAvailableViewDataTables() { + $cache = Cache::getTransientCache(); + $cacheId = 'ViewDataTable.getAvailableViewDataTables'; + $dataTables = $cache->fetch($cacheId); + + if (!empty($dataTables)) { + return $dataTables; + } + $klassToExtend = '\\Piwik\\Plugin\\ViewDataTable'; /** @var string[] $visualizations */ @@ -107,6 +116,8 @@ public static function getAvailableViewDataTables() $result[$vizId] = $viz; } + $cache->save($cacheId, $result); + return $result; } From 832ecfb6041c47e6de618d484b1ba6d678ebdf71 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 2 Feb 2015 02:02:42 +0000 Subject: [PATCH 4/4] Cache generated date range periods. This is a bit experimental and I need to see whether any test will fail. It takes a long time to generate subperiods for period=range. Eg requesting date=2015-01-04,2015-02-02&module=Referrers&action=getSearchEngines&period=range took about 700ms for each request of 1500ms in total. It now takes only 40ms which means it is 30-40% faster. With the other optimizations the request in total does now only take 700-800ms instead of 1500ms. --- core/Period/Range.php | 62 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/core/Period/Range.php b/core/Period/Range.php index d88a5e2da74..4f1c5b3c7f9 100644 --- a/core/Period/Range.php +++ b/core/Period/Range.php @@ -9,6 +9,7 @@ namespace Piwik\Period; use Exception; +use Piwik\Cache; use Piwik\Common; use Piwik\Date; use Piwik\Period; @@ -32,6 +33,11 @@ class Range extends Period protected $label = 'range'; protected $today; + /** + * @var null|Date + */ + protected $defaultEndDate; + /** * Constructor. * @@ -56,6 +62,41 @@ public function __construct($strPeriod, $strDate, $timezone = 'UTC', $today = fa $this->today = $today; } + private function getCache() + { + return Cache::getTransientCache(); + } + + private function getCacheId() + { + $end = ''; + if ($this->defaultEndDate) { + $end = $this->defaultEndDate->getTimestamp(); + } + + $today = $this->today->getTimestamp(); + + return 'range' . $this->strPeriod . $this->strDate . $this->timezone . $end . $today; + } + + private function loadAllFromCache() + { + $range = $this->getCache()->fetch($this->getCacheId()); + + if (!empty($range)) { + foreach ($range as $key => $val) { + $this->$key = $val; + } + } + } + + private function cacheAll() + { + $props = get_object_vars($this); + + $this->getCache()->save($this->getCacheId(), $props); + } + /** * Returns the current period as a localized short string. * @@ -156,6 +197,12 @@ protected function generate() return; } + $this->loadAllFromCache(); + + if ($this->subperiodsProcessed) { + return; + } + parent::generate(); if (preg_match('/(last|previous)([0-9]*)/', $this->strDate, $regs)) { @@ -214,6 +261,7 @@ protected function generate() if ($this->strPeriod != 'range') { $this->fillArraySubPeriods($startDate, $endDate, $this->strPeriod); + $this->cacheAll(); return; } @@ -221,6 +269,7 @@ protected function generate() // When period=range, we want End Date to be the actual specified end date, // rather than the end of the month / week / whatever is used for processing this range $this->endDate = $endDate; + $this->cacheAll(); } /** @@ -462,4 +511,17 @@ private function isEndOfWeekLaterThanEndDate($endDate, $endOfWeek) return $isEndOfWeekLaterThanEndDate; } + + /** + * Returns the date range string comprising two dates + * + * @return string eg, `'2012-01-01,2012-01-31'`. + */ + public function getRangeString() + { + $dateStart = $this->getDateStart(); + $dateEnd = $this->getDateEnd(); + + return $dateStart->toString("Y-m-d") . "," . $dateEnd->toString("Y-m-d"); + } }