From 387e2692b87b98599b093cabbb1b215aaacab870 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Thu, 2 Jul 2020 10:05:31 +0200 Subject: [PATCH 01/18] use subqueries for segments using not contains / not equals operator not on log_visit table --- core/Segment.php | 62 +++++++++++++++++++++++++++++- core/Segment/SegmentExpression.php | 11 +++++- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index 26f6de430b1..b004864aad9 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -12,6 +12,7 @@ use Piwik\API\Request; use Piwik\ArchiveProcessor\Rules; use Piwik\Container\StaticContainer; +use Piwik\DataAccess\LogAggregator; use Piwik\DataAccess\LogQueryBuilder; use Piwik\Plugins\SegmentEditor\SegmentEditor; use Piwik\Segment\SegmentExpression; @@ -208,8 +209,41 @@ protected function initializeSegment($string, $idSites) // and apply a filter to the value to match if necessary (to map DB fields format) $cleanedExpressions = array(); foreach ($expressions as $expression) { + $cleanedExpression = null; $operand = $expression[SegmentExpression::INDEX_OPERAND]; - $cleanedExpression = $this->getCleanedExpression($operand); + $name = $operand[SegmentExpression::INDEX_OPERAND_NAME]; + + // Build subqueries for segments that are not on log_visit table but use !@ or != as operator + // This is required to ensure segments like actionUrl!@value really do not include any visit having an action containing `value` + if (!$this->isVisitSegment($name) && in_array($operand[SegmentExpression::INDEX_OPERAND_OPERATOR], [ + SegmentExpression::MATCH_DOES_NOT_CONTAIN, + SegmentExpression::MATCH_NOT_EQUAL + ])) { + $operator = $operand[SegmentExpression::INDEX_OPERAND_OPERATOR] === SegmentExpression::MATCH_DOES_NOT_CONTAIN ? SegmentExpression::MATCH_CONTAINS : SegmentExpression::MATCH_EQUAL; + $stringSegment = $operand[SegmentExpression::INDEX_OPERAND_NAME] . $operator . $operand[SegmentExpression::INDEX_OPERAND_VALUE]; + $segmentObj = new Segment($stringSegment, $idSites); + + $date = Common::getRequestVar('date', false); + $periodStr = Common::getRequestVar('period', false); + $period = Period\Factory::build($periodStr, $date); + + $params = new ArchiveProcessor\Parameters(new Site(is_array($idSites) ? reset($idSites) : $idSites), $period, $segmentObj); + $logAggregator = new LogAggregator($params); + $select = 'log_visit.idvisit'; + $from = 'log_visit'; + $where = $logAggregator->getWhereStatement('log_visit', 'visit_last_action_time'); + $query = $logAggregator->generateQuery($select, $from, $where, 'log_visit.idvisit', ''); + + $cleanedExpression = [ + SegmentExpression::INDEX_OPERAND_NAME => 'log_visit.idvisit', + SegmentExpression::INDEX_OPERAND_OPERATOR => SegmentExpression::MATCH_ACTIONS_NOT_CONTAINS, + SegmentExpression::INDEX_OPERAND_VALUE => $query + ]; + } + + if (empty($cleanedExpression)) { + $cleanedExpression = $this->getCleanedExpression($operand); + } $expression[SegmentExpression::INDEX_OPERAND] = $cleanedExpression; $cleanedExpressions[] = $expression; } @@ -226,7 +260,11 @@ private function getExpressionsWithUnionsResolved($expressions) $availableSegment = $this->getSegmentByName($name); - if (!empty($availableSegment['unionOfSegments'])) { + // We leave segments using !@ and != operands untouched for segments not on log_visit table as they will be build using a subquery + if ((!in_array($operand[SegmentExpression::INDEX_OPERAND_OPERATOR], [ + SegmentExpression::MATCH_DOES_NOT_CONTAIN, + SegmentExpression::MATCH_NOT_EQUAL + ]) || $this->isVisitSegment($name)) && !empty($availableSegment['unionOfSegments'])) { $count = 0; foreach ($availableSegment['unionOfSegments'] as $segmentNameOfUnion) { $count++; @@ -252,6 +290,26 @@ private function getExpressionsWithUnionsResolved($expressions) return $expressionsWithUnions; } + private function isVisitSegment($name) + { + $availableSegment = $this->getSegmentByName($name); + + $isVisitSegment = false; + if (!empty($availableSegment['unionOfSegments'])) { + foreach ($availableSegment['unionOfSegments'] as $segmentNameOfUnion) { + $unionSegment = $this->getSegmentByName($segmentNameOfUnion); + if (strpos($unionSegment['sqlSegment'], 'log_visit.') === 0) { + $isVisitSegment = true; + break; + } + } + } else if (strpos($availableSegment['sqlSegment'], 'log_visit.') === 0) { + $isVisitSegment = true; + } + + return $isVisitSegment; + } + /** * Returns `true` if the segment is empty, `false` if otherwise. */ diff --git a/core/Segment/SegmentExpression.php b/core/Segment/SegmentExpression.php index dc710197074..8e9462a7085 100644 --- a/core/Segment/SegmentExpression.php +++ b/core/Segment/SegmentExpression.php @@ -42,6 +42,7 @@ class SegmentExpression // Special case, since we look up Page URLs/Page titles in a sub SQL query const MATCH_ACTIONS_CONTAINS = 'IN'; + const MATCH_ACTIONS_NOT_CONTAINS = 'NOTIN'; const INDEX_BOOL_OPERATOR = 0; const INDEX_OPERAND = 1; @@ -289,6 +290,14 @@ protected function getSqlMatchFromDefinition($def, &$availableTables) $sqlMatch = '%s IN (' . $value['SQL'] . ')'; $value = $value['bind']; break; + case self::MATCH_ACTIONS_NOT_CONTAINS: + // this match type is not accessible from the outside + // (it won't be matched in self::parseSubExpressions()) + // it can be used internally to inject sub-expressions into the query. + // see Segment::getCleanedExpression() + $sqlMatch = '%s NOT IN (' . $value['sql'] . ')'; + $value = $value['bind']; + break; default: throw new Exception("Filter contains the match type '" . $matchType . "' which is not supported"); break; @@ -298,7 +307,7 @@ protected function getSqlMatchFromDefinition($def, &$availableTables) $alsoMatchNULLValues = $alsoMatchNULLValues && !empty($value); $sqlMatch = str_replace('%s', $field, $sqlMatch); - if ($matchType === self::MATCH_ACTIONS_CONTAINS + if ($matchType === self::MATCH_ACTIONS_CONTAINS || $matchType === self::MATCH_ACTIONS_NOT_CONTAINS || is_null($value) ) { $sqlExpression = "( $sqlMatch )"; From 246e07264003336aa6c641815486f0f959ea81e5 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Thu, 2 Jul 2020 14:28:43 +0200 Subject: [PATCH 02/18] remove duplicate code --- core/Segment.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index b004864aad9..8628e3d6ddb 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -215,10 +215,7 @@ protected function initializeSegment($string, $idSites) // Build subqueries for segments that are not on log_visit table but use !@ or != as operator // This is required to ensure segments like actionUrl!@value really do not include any visit having an action containing `value` - if (!$this->isVisitSegment($name) && in_array($operand[SegmentExpression::INDEX_OPERAND_OPERATOR], [ - SegmentExpression::MATCH_DOES_NOT_CONTAIN, - SegmentExpression::MATCH_NOT_EQUAL - ])) { + if ($this->doesSegmentNeedSubquery($operand[SegmentExpression::INDEX_OPERAND_OPERATOR], $name)) { $operator = $operand[SegmentExpression::INDEX_OPERAND_OPERATOR] === SegmentExpression::MATCH_DOES_NOT_CONTAIN ? SegmentExpression::MATCH_CONTAINS : SegmentExpression::MATCH_EQUAL; $stringSegment = $operand[SegmentExpression::INDEX_OPERAND_NAME] . $operator . $operand[SegmentExpression::INDEX_OPERAND_VALUE]; $segmentObj = new Segment($stringSegment, $idSites); @@ -261,10 +258,7 @@ private function getExpressionsWithUnionsResolved($expressions) $availableSegment = $this->getSegmentByName($name); // We leave segments using !@ and != operands untouched for segments not on log_visit table as they will be build using a subquery - if ((!in_array($operand[SegmentExpression::INDEX_OPERAND_OPERATOR], [ - SegmentExpression::MATCH_DOES_NOT_CONTAIN, - SegmentExpression::MATCH_NOT_EQUAL - ]) || $this->isVisitSegment($name)) && !empty($availableSegment['unionOfSegments'])) { + if (!$this->doesSegmentNeedSubquery($operand[SegmentExpression::INDEX_OPERAND_OPERATOR], $name) && !empty($availableSegment['unionOfSegments'])) { $count = 0; foreach ($availableSegment['unionOfSegments'] as $segmentNameOfUnion) { $count++; @@ -310,6 +304,14 @@ private function isVisitSegment($name) return $isVisitSegment; } + private function doesSegmentNeedSubquery($operator, $segmentName) + { + return in_array($operator, [ + SegmentExpression::MATCH_DOES_NOT_CONTAIN, + SegmentExpression::MATCH_NOT_EQUAL + ]) && !$this->isVisitSegment($segmentName); + } + /** * Returns `true` if the segment is empty, `false` if otherwise. */ From 378c49b609899e96379ab6cdf540f6e75b7dddc7 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Thu, 2 Jul 2020 16:35:29 +0200 Subject: [PATCH 03/18] simplify code --- core/Segment.php | 54 ++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index 8628e3d6ddb..b032d19ae7e 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -209,39 +209,8 @@ protected function initializeSegment($string, $idSites) // and apply a filter to the value to match if necessary (to map DB fields format) $cleanedExpressions = array(); foreach ($expressions as $expression) { - $cleanedExpression = null; $operand = $expression[SegmentExpression::INDEX_OPERAND]; - $name = $operand[SegmentExpression::INDEX_OPERAND_NAME]; - - // Build subqueries for segments that are not on log_visit table but use !@ or != as operator - // This is required to ensure segments like actionUrl!@value really do not include any visit having an action containing `value` - if ($this->doesSegmentNeedSubquery($operand[SegmentExpression::INDEX_OPERAND_OPERATOR], $name)) { - $operator = $operand[SegmentExpression::INDEX_OPERAND_OPERATOR] === SegmentExpression::MATCH_DOES_NOT_CONTAIN ? SegmentExpression::MATCH_CONTAINS : SegmentExpression::MATCH_EQUAL; - $stringSegment = $operand[SegmentExpression::INDEX_OPERAND_NAME] . $operator . $operand[SegmentExpression::INDEX_OPERAND_VALUE]; - $segmentObj = new Segment($stringSegment, $idSites); - - $date = Common::getRequestVar('date', false); - $periodStr = Common::getRequestVar('period', false); - $period = Period\Factory::build($periodStr, $date); - - $params = new ArchiveProcessor\Parameters(new Site(is_array($idSites) ? reset($idSites) : $idSites), $period, $segmentObj); - $logAggregator = new LogAggregator($params); - $select = 'log_visit.idvisit'; - $from = 'log_visit'; - $where = $logAggregator->getWhereStatement('log_visit', 'visit_last_action_time'); - $query = $logAggregator->generateQuery($select, $from, $where, 'log_visit.idvisit', ''); - - $cleanedExpression = [ - SegmentExpression::INDEX_OPERAND_NAME => 'log_visit.idvisit', - SegmentExpression::INDEX_OPERAND_OPERATOR => SegmentExpression::MATCH_ACTIONS_NOT_CONTAINS, - SegmentExpression::INDEX_OPERAND_VALUE => $query - ]; - } - - if (empty($cleanedExpression)) { - $cleanedExpression = $this->getCleanedExpression($operand); - } - $expression[SegmentExpression::INDEX_OPERAND] = $cleanedExpression; + $expression[SegmentExpression::INDEX_OPERAND] = $this->getCleanedExpression($operand); $cleanedExpressions[] = $expression; } @@ -357,6 +326,27 @@ protected function getCleanedExpression($expression) $segment = $this->getSegmentByName($name); $sqlName = $segment['sqlSegment']; + // Build subqueries for segments that are not on log_visit table but use !@ or != as operator + // This is required to ensure segments like actionUrl!@value really do not include any visit having an action containing `value` + if ($this->doesSegmentNeedSubquery($matchType, $name)) { + $operator = $matchType === SegmentExpression::MATCH_DOES_NOT_CONTAIN ? SegmentExpression::MATCH_CONTAINS : SegmentExpression::MATCH_EQUAL; + $stringSegment = $name . $operator . $value; + $segmentObj = new Segment($stringSegment, $this->idSites); + + $date = Common::getRequestVar('date', false); + $periodStr = Common::getRequestVar('period', false); + $period = Period\Factory::build($periodStr, $date); + + $params = new ArchiveProcessor\Parameters(new Site(is_array($this->idSites) ? reset($this->idSites) : $this->idSites), $period, $segmentObj); + $logAggregator = new LogAggregator($params); + $select = 'log_visit.idvisit'; + $from = 'log_visit'; + $where = $logAggregator->getWhereStatement('log_visit', 'visit_last_action_time'); + $query = $logAggregator->generateQuery($select, $from, $where, 'log_visit.idvisit', ''); + + return ['log_visit.idvisit', SegmentExpression::MATCH_ACTIONS_NOT_CONTAINS, $query]; + } + if ($matchType != SegmentExpression::MATCH_IS_NOT_NULL_NOR_EMPTY && $matchType != SegmentExpression::MATCH_IS_NULL_OR_EMPTY) { From c1cc31ff4eb132cef77a57a9edaa274c164f93e8 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Thu, 2 Jul 2020 17:25:44 +0200 Subject: [PATCH 04/18] inject period --- core/Segment.php | 24 ++++++++++++++++++------ plugins/Live/Model.php | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index b032d19ae7e..d1af04f5cc6 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -76,6 +76,11 @@ class Segment */ protected $idSites = null; + /** + * @var Period + */ + protected $period = null; + /** * @var LogQueryBuilder */ @@ -97,9 +102,10 @@ class Segment * @param string $segmentCondition The segment condition, eg, `'browserCode=ff;countryCode=CA'`. * @param array $idSites The list of sites the segment will be used with. Some segments are * dependent on the site, such as goal segments. + * @param Period|null $period * @throws */ - public function __construct($segmentCondition, $idSites) + public function __construct($segmentCondition, $idSites, $period = null) { $this->segmentQueryBuilder = StaticContainer::get('Piwik\DataAccess\LogQueryBuilder'); @@ -112,6 +118,16 @@ public function __construct($segmentCondition, $idSites) $this->originalString = $segmentCondition; + if ($period instanceof Period) { + $this->period = $period; + } + + if (empty($this->period)) { + $date = Common::getRequestVar('date', false); + $periodStr = Common::getRequestVar('period', false); + $this->period = Period\Factory::build($periodStr, $date); + } + // The segment expression can be urlencoded. Unfortunately, both the encoded and decoded versions // can usually be parsed successfully. To pick the right one, we try both and pick the one w/ more // successfully parsed subexpressions. @@ -333,11 +349,7 @@ protected function getCleanedExpression($expression) $stringSegment = $name . $operator . $value; $segmentObj = new Segment($stringSegment, $this->idSites); - $date = Common::getRequestVar('date', false); - $periodStr = Common::getRequestVar('period', false); - $period = Period\Factory::build($periodStr, $date); - - $params = new ArchiveProcessor\Parameters(new Site(is_array($this->idSites) ? reset($this->idSites) : $this->idSites), $period, $segmentObj); + $params = new ArchiveProcessor\Parameters(new Site(is_array($this->idSites) ? reset($this->idSites) : $this->idSites), $this->period, $segmentObj); $logAggregator = new LogAggregator($params); $select = 'log_visit.idvisit'; $from = 'log_visit'; diff --git a/plugins/Live/Model.php b/plugins/Live/Model.php index 9590cf00e3a..1ad6ce61e47 100644 --- a/plugins/Live/Model.php +++ b/plugins/Live/Model.php @@ -466,7 +466,7 @@ public function makeLogVisitsQueryString($idSite, $startDate, $endDate, $segment $filterSortOrder = 'DESC'; } - $segment = new Segment($segment, $idSite); + $segment = new Segment($segment, $idSite, Period\Factory::build('day', implode(',', array_filter([$startDate,$endDate])))); // Subquery to use the indexes for ORDER BY $select = "log_visit.*"; From 0d5aa3c4088fe44af870c818aa3bf64b8aed512c Mon Sep 17 00:00:00 2001 From: sgiehl Date: Fri, 3 Jul 2020 09:47:38 +0200 Subject: [PATCH 05/18] apply some review feedback --- core/Segment.php | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index d1af04f5cc6..38c05d4476f 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -212,6 +212,10 @@ protected function initializeSegment($string, $idSites) $string = substr($string, 0, self::SEGMENT_TRUNCATE_LIMIT); $this->string = $string; + + if (!is_array($idSites)) { + $idSites = [$idSites]; + } $this->idSites = $idSites; $segment = new SegmentExpression($string); $this->segmentExpression = $segment; @@ -273,20 +277,18 @@ private function isVisitSegment($name) { $availableSegment = $this->getSegmentByName($name); - $isVisitSegment = false; if (!empty($availableSegment['unionOfSegments'])) { foreach ($availableSegment['unionOfSegments'] as $segmentNameOfUnion) { $unionSegment = $this->getSegmentByName($segmentNameOfUnion); if (strpos($unionSegment['sqlSegment'], 'log_visit.') === 0) { - $isVisitSegment = true; - break; + return true; } } } else if (strpos($availableSegment['sqlSegment'], 'log_visit.') === 0) { - $isVisitSegment = true; + return true; } - return $isVisitSegment; + return false; } private function doesSegmentNeedSubquery($operator, $segmentName) @@ -297,6 +299,17 @@ private function doesSegmentNeedSubquery($operator, $segmentName) ]) && !$this->isVisitSegment($segmentName); } + private function getInvertedOperatorForSubQuery($operator) + { + if ($operator === SegmentExpression::MATCH_DOES_NOT_CONTAIN) { + return SegmentExpression::MATCH_CONTAINS; + } else if ($operator === SegmentExpression::MATCH_NOT_EQUAL) { + return SegmentExpression::MATCH_EQUAL; + } + + throw new Exception("Operator not support for subqueries"); + } + /** * Returns `true` if the segment is empty, `false` if otherwise. */ @@ -322,9 +335,6 @@ public function willBeArchived() } $idSites = $this->idSites; - if (!is_array($idSites)) { - $idSites = array($this->idSites); - } return Rules::isRequestAuthorizedToArchive() || Rules::isBrowserArchivingAvailableForSegments() @@ -345,12 +355,16 @@ protected function getCleanedExpression($expression) // Build subqueries for segments that are not on log_visit table but use !@ or != as operator // This is required to ensure segments like actionUrl!@value really do not include any visit having an action containing `value` if ($this->doesSegmentNeedSubquery($matchType, $name)) { - $operator = $matchType === SegmentExpression::MATCH_DOES_NOT_CONTAIN ? SegmentExpression::MATCH_CONTAINS : SegmentExpression::MATCH_EQUAL; + $operator = $this->getInvertedOperatorForSubQuery($matchType); $stringSegment = $name . $operator . $value; $segmentObj = new Segment($stringSegment, $this->idSites); - $params = new ArchiveProcessor\Parameters(new Site(is_array($this->idSites) ? reset($this->idSites) : $this->idSites), $this->period, $segmentObj); + // use the first site to be able to initialize the Parameters object, but overwrite it for LogAggregator afterwards + $site = new Site(reset($this->idSites)); + $params = new ArchiveProcessor\Parameters($site, $this->period, $segmentObj); $logAggregator = new LogAggregator($params); + $logAggregator->setSites($this->idSites); + $select = 'log_visit.idvisit'; $from = 'log_visit'; $where = $logAggregator->getWhereStatement('log_visit', 'visit_last_action_time'); From e772ebd0b42fa8421b408edeb13ee873fd35ebd4 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Fri, 3 Jul 2020 14:24:30 +0200 Subject: [PATCH 06/18] do not use subquery if no period is available --- core/Segment.php | 6 ++++-- plugins/Live/Model.php | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index 38c05d4476f..275ead8ed8f 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -125,7 +125,9 @@ public function __construct($segmentCondition, $idSites, $period = null) if (empty($this->period)) { $date = Common::getRequestVar('date', false); $periodStr = Common::getRequestVar('period', false); - $this->period = Period\Factory::build($periodStr, $date); + if ($date && $periodStr) { + $this->period = Period\Factory::build($periodStr, $date); + } } // The segment expression can be urlencoded. Unfortunately, both the encoded and decoded versions @@ -293,7 +295,7 @@ private function isVisitSegment($name) private function doesSegmentNeedSubquery($operator, $segmentName) { - return in_array($operator, [ + return $this->period instanceof Period && in_array($operator, [ SegmentExpression::MATCH_DOES_NOT_CONTAIN, SegmentExpression::MATCH_NOT_EQUAL ]) && !$this->isVisitSegment($segmentName); diff --git a/plugins/Live/Model.php b/plugins/Live/Model.php index 1ad6ce61e47..930ed097fb6 100644 --- a/plugins/Live/Model.php +++ b/plugins/Live/Model.php @@ -466,7 +466,10 @@ public function makeLogVisitsQueryString($idSite, $startDate, $endDate, $segment $filterSortOrder = 'DESC'; } - $segment = new Segment($segment, $idSite, Period\Factory::build('day', implode(',', array_filter([$startDate,$endDate])))); + $strStart = $startDate instanceof Date ? date('Y-m-d', $startDate->getTimestampUTC()) : $startDate; + $strEnd = $endDate instanceof Date ? date('Y-m-d', $endDate->getTimestampUTC()) : $endDate; + + $segment = new Segment($segment, $idSite, $period = Period\Factory::build('day', implode(',', array_filter([$strStart,$strEnd])))); // Subquery to use the indexes for ORDER BY $select = "log_visit.*"; From 07c47b319bfaff58949801d801e12cc01be0429f Mon Sep 17 00:00:00 2001 From: sgiehl Date: Mon, 6 Jul 2020 09:46:03 +0200 Subject: [PATCH 07/18] Avoid using a period in favor of start and end date and no longer use LogAggregator to build segment subquery --- core/Segment.php | 49 +++++++++++++++++++++++------------------- plugins/Live/Model.php | 12 +++++------ 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index 275ead8ed8f..17cacb03013 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -12,7 +12,6 @@ use Piwik\API\Request; use Piwik\ArchiveProcessor\Rules; use Piwik\Container\StaticContainer; -use Piwik\DataAccess\LogAggregator; use Piwik\DataAccess\LogQueryBuilder; use Piwik\Plugins\SegmentEditor\SegmentEditor; use Piwik\Segment\SegmentExpression; @@ -77,9 +76,14 @@ class Segment protected $idSites = null; /** - * @var Period + * @var Date */ - protected $period = null; + protected $startDate = null; + + /** + * @var Date + */ + protected $endDate = null; /** * @var LogQueryBuilder @@ -102,10 +106,11 @@ class Segment * @param string $segmentCondition The segment condition, eg, `'browserCode=ff;countryCode=CA'`. * @param array $idSites The list of sites the segment will be used with. Some segments are * dependent on the site, such as goal segments. - * @param Period|null $period + * @param Date|null $startDate + * @param Date|null $endDate * @throws */ - public function __construct($segmentCondition, $idSites, $period = null) + public function __construct($segmentCondition, $idSites, Date $startDate = null, Date $endDate = null) { $this->segmentQueryBuilder = StaticContainer::get('Piwik\DataAccess\LogQueryBuilder'); @@ -118,16 +123,12 @@ public function __construct($segmentCondition, $idSites, $period = null) $this->originalString = $segmentCondition; - if ($period instanceof Period) { - $this->period = $period; + if ($startDate instanceof Date) { + $this->startDate = $startDate; } - if (empty($this->period)) { - $date = Common::getRequestVar('date', false); - $periodStr = Common::getRequestVar('period', false); - if ($date && $periodStr) { - $this->period = Period\Factory::build($periodStr, $date); - } + if ($endDate instanceof Date) { + $this->endDate = $endDate; } // The segment expression can be urlencoded. Unfortunately, both the encoded and decoded versions @@ -295,7 +296,7 @@ private function isVisitSegment($name) private function doesSegmentNeedSubquery($operator, $segmentName) { - return $this->period instanceof Period && in_array($operator, [ + return in_array($operator, [ SegmentExpression::MATCH_DOES_NOT_CONTAIN, SegmentExpression::MATCH_NOT_EQUAL ]) && !$this->isVisitSegment($segmentName); @@ -361,16 +362,20 @@ protected function getCleanedExpression($expression) $stringSegment = $name . $operator . $value; $segmentObj = new Segment($stringSegment, $this->idSites); - // use the first site to be able to initialize the Parameters object, but overwrite it for LogAggregator afterwards - $site = new Site(reset($this->idSites)); - $params = new ArchiveProcessor\Parameters($site, $this->period, $segmentObj); - $logAggregator = new LogAggregator($params); - $logAggregator->setSites($this->idSites); - $select = 'log_visit.idvisit'; $from = 'log_visit'; - $where = $logAggregator->getWhereStatement('log_visit', 'visit_last_action_time'); - $query = $logAggregator->generateQuery($select, $from, $where, 'log_visit.idvisit', ''); + $datetimeField = 'visit_last_action_time'; + $where = "$from.idsite IN (". Common::getSqlStringFieldsArray($this->idSites) . ")"; + $bind = $this->idSites; + if ($this->startDate instanceof Date) { + $where .= " AND $from.$datetimeField >= ?"; + $bind[] = $this->startDate->toString(Date::DATE_TIME_FORMAT); + } + if ($this->endDate instanceof Date) { + $where .= " AND $from.$datetimeField <= ?"; + $bind[] = $this->endDate->toString(Date::DATE_TIME_FORMAT); + } + $query = $segmentObj->getSelectQuery($select, $from, $where, $bind); return ['log_visit.idvisit', SegmentExpression::MATCH_ACTIONS_NOT_CONTAINS, $query]; } diff --git a/plugins/Live/Model.php b/plugins/Live/Model.php index 930ed097fb6..9f5815e0052 100644 --- a/plugins/Live/Model.php +++ b/plugins/Live/Model.php @@ -359,11 +359,12 @@ private function getLastMinutesCounterForQuery($idSite, $lastMinutes, $segment, $now = $now ?: time(); $bind = $idSites; - $bind[] = Date::factory($now - $lastMinutes * 60)->toString('Y-m-d H:i:s'); + $startDate = Date::factory($now - $lastMinutes * 60)->toString('Y-m-d H:i:s'); + $bind[] = $startDate; $where = $whereIdSites . "AND " . $where; - $segment = new Segment($segment, $idSite); + $segment = new Segment($segment, $idSite, $startDate, $endDate = null); $query = $segment->getSelectQuery($select, $from, $where, $bind); $numVisitors = Db::getReader()->fetchOne($query['sql'], $query['bind']); @@ -428,7 +429,7 @@ public function queryAdjacentVisitorId($idSite, $visitorId, $visitLastActionTime $orderBy = "MAX(log_visit.visit_last_action_time) $orderByDir"; $groupBy = "log_visit.idvisitor"; - $segment = new Segment($segment, $idSite); + $segment = new Segment($segment, $idSite, $dateOneDayAgo, $dateOneDayInFuture); $queryInfo = $segment->getSelectQuery($select, $from, $where, $whereBind, $orderBy, $groupBy); $sql = "SELECT sub.idvisitor, sub.visit_last_action_time FROM ({$queryInfo['sql']}) as sub @@ -466,10 +467,7 @@ public function makeLogVisitsQueryString($idSite, $startDate, $endDate, $segment $filterSortOrder = 'DESC'; } - $strStart = $startDate instanceof Date ? date('Y-m-d', $startDate->getTimestampUTC()) : $startDate; - $strEnd = $endDate instanceof Date ? date('Y-m-d', $endDate->getTimestampUTC()) : $endDate; - - $segment = new Segment($segment, $idSite, $period = Period\Factory::build('day', implode(',', array_filter([$strStart,$strEnd])))); + $segment = new Segment($segment, $idSite, $startDate, $endDate); // Subquery to use the indexes for ORDER BY $select = "log_visit.*"; From e626c3c40ad7b5bbdff074c77a39731c53eb3a3e Mon Sep 17 00:00:00 2001 From: sgiehl Date: Mon, 6 Jul 2020 10:14:16 +0200 Subject: [PATCH 08/18] inject dates on some more places --- core/ArchiveProcessor.php | 2 +- core/CronArchive.php | 8 ++++---- plugins/Live/Model.php | 4 ++-- plugins/SegmentEditor/SegmentEditor.php | 2 +- plugins/Transitions/API.php | 2 +- tests/PHPUnit/Integration/Archive/ChunksTest.php | 2 +- tests/PHPUnit/Integration/ArchiveProcessingTest.php | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php index af8feb6716e..202eb5f04de 100644 --- a/core/ArchiveProcessor.php +++ b/core/ArchiveProcessor.php @@ -650,7 +650,7 @@ public function processDependentArchive($plugin, $segment) return; } - $newSegment = new Segment($newSegment, $idSites); + $newSegment = new Segment($newSegment, $idSites, $params->getDateStart(), $params->getDateEnd()); if (ArchiveProcessor\Rules::isSegmentPreProcessed($idSites, $newSegment)) { // will be processed anyway return; diff --git a/core/CronArchive.php b/core/CronArchive.php index cbe9a34234d..b52fa24ab85 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -589,7 +589,7 @@ private function launchArchivingFor($archives) $idSite = $archive['idsite']; $dateStr = $archive['period'] == Range::PERIOD_ID ? ($archive['date1'] . ',' . $archive['date2']) : $archive['date1']; $period = PeriodFactory::build($this->periodIdsToLabels[$archive['period']], $dateStr); - $params = new Parameters(new Site($idSite), $period, new Segment($segment, [$idSite])); + $params = new Parameters(new Site($idSite), $period, new Segment($segment, [$idSite], $period->getDateStart(), $period->getDateEnd())); $loader = new Loader($params); if ($loader->canSkipThisArchive()) { @@ -941,7 +941,7 @@ public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain() continue; } - $params = new Parameters(new Site($idSite), $period, new Segment('', [$idSite])); + $params = new Parameters(new Site($idSite), $period, new Segment('', [$idSite], $period->getDateStart(), $period->getDateEnd())); if ($this->isThereExistingValidPeriod($params)) { $this->logger->info(' Found usable archive for custom date range {date} for site {idSite}, skipping archiving.', ['date' => $date, 'idSite' => $idSite]); continue; @@ -989,7 +989,7 @@ private function invalidateRecentDate($dateStr) $date = Date::factory($dateStr); $period = PeriodFactory::build('day', $date); - $params = new Parameters(new Site($idSite), $period, new Segment('', [$idSite])); + $params = new Parameters(new Site($idSite), $period, new Segment('', [$idSite], $period->getDateStart(), $period->getDateEnd())); if ($this->isThereExistingValidPeriod($params, $isYesterday)) { $this->logger->debug(" Found existing valid archive for $dateStr, skipping invalidation..."); continue; @@ -1420,7 +1420,7 @@ private function canSkipArchiveBecauseNoPoint(array $invalidatedArchive) $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1']; $period = PeriodFactory::build($periodLabel, $dateStr); - $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']]); + $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']], $period->getDateStart(), $period->getDateEnd()); $params = new Parameters($site, $period, $segment); diff --git a/plugins/Live/Model.php b/plugins/Live/Model.php index 9f5815e0052..023d7169832 100644 --- a/plugins/Live/Model.php +++ b/plugins/Live/Model.php @@ -359,8 +359,8 @@ private function getLastMinutesCounterForQuery($idSite, $lastMinutes, $segment, $now = $now ?: time(); $bind = $idSites; - $startDate = Date::factory($now - $lastMinutes * 60)->toString('Y-m-d H:i:s'); - $bind[] = $startDate; + $startDate = Date::factory($now - $lastMinutes * 60); + $bind[] = $startDate->toString('Y-m-d H:i:s'); $where = $whereIdSites . "AND " . $where; diff --git a/plugins/SegmentEditor/SegmentEditor.php b/plugins/SegmentEditor/SegmentEditor.php index de89ca2c98d..59ac72ee9c5 100644 --- a/plugins/SegmentEditor/SegmentEditor.php +++ b/plugins/SegmentEditor/SegmentEditor.php @@ -201,12 +201,12 @@ private function getSegmentIfIsUnprocessed() if (empty($segment)) { return null; } - $segment = new Segment($segment, [$idSite]); // get period $date = Common::getRequestVar('date', false); $periodStr = Common::getRequestVar('period', false); $period = Period\Factory::build($periodStr, $date); + $segment = new Segment($segment, [$idSite], $period->getDateStart(), $period->getDateEnd()); // check if archiving is enabled. if so, the segment should have been processed. $isArchivingDisabled = Rules::isArchivingDisabledFor([$idSite], $segment, $period); diff --git a/plugins/Transitions/API.php b/plugins/Transitions/API.php index 39435541a27..a517e8653d2 100644 --- a/plugins/Transitions/API.php +++ b/plugins/Transitions/API.php @@ -73,9 +73,9 @@ public function getTransitionsForAction($actionName, $actionType, $idSite, $peri } // prepare log aggregator - $segment = new Segment($segment, $idSite); $site = new Site($idSite); $period = Period\Factory::build($period, $date); + $segment = new Segment($segment, $idSite, $period->getDateStart(), $period->getDateEnd()); $params = new ArchiveProcessor\Parameters($site, $period, $segment); $logAggregator = new LogAggregator($params); diff --git a/tests/PHPUnit/Integration/Archive/ChunksTest.php b/tests/PHPUnit/Integration/Archive/ChunksTest.php index 96a598128e1..3a8c7bb71af 100644 --- a/tests/PHPUnit/Integration/Archive/ChunksTest.php +++ b/tests/PHPUnit/Integration/Archive/ChunksTest.php @@ -129,7 +129,7 @@ private function createArchiveProcessorParameters() { $oPeriod = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', $this->date); - $segment = new Segment(false, array(1)); + $segment = new Segment(false, array(1), $oPeriod->getDateStart(), $oPeriod->getDateEnd()); $params = new Parameters(new Site(1), $oPeriod, $segment); return $params; diff --git a/tests/PHPUnit/Integration/ArchiveProcessingTest.php b/tests/PHPUnit/Integration/ArchiveProcessingTest.php index 2db256a1419..9e63b2536dd 100644 --- a/tests/PHPUnit/Integration/ArchiveProcessingTest.php +++ b/tests/PHPUnit/Integration/ArchiveProcessingTest.php @@ -89,7 +89,7 @@ private function _createArchiveProcessor($periodLabel, $dateLabel, $siteTimezone $site = $this->_createWebsite($siteTimezone); $date = Date::factory($dateLabel); $period = Period\Factory::build($periodLabel, $date); - $segment = new Segment('', $site->getId()); + $segment = new Segment('', $site->getId(), $period->getDateStart(), $period->getDateEnd()); $params = new ArchiveProcessor\Parameters($site, $period, $segment); return new ArchiveProcessorTest($params); From 195da46f1ebb335bf12b7cc1f635d1acb7ac850b Mon Sep 17 00:00:00 2001 From: sgiehl Date: Mon, 6 Jul 2020 11:53:47 +0200 Subject: [PATCH 09/18] Adds some tests --- core/Segment.php | 12 ++- tests/PHPUnit/Integration/SegmentTest.php | 94 +++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index 17cacb03013..354762acabe 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -216,7 +216,9 @@ protected function initializeSegment($string, $idSites) $this->string = $string; - if (!is_array($idSites)) { + if (empty($idSites)) { + $idSites = []; + } else if (!is_array($idSites)) { $idSites = [$idSites]; } $this->idSites = $idSites; @@ -365,8 +367,12 @@ protected function getCleanedExpression($expression) $select = 'log_visit.idvisit'; $from = 'log_visit'; $datetimeField = 'visit_last_action_time'; - $where = "$from.idsite IN (". Common::getSqlStringFieldsArray($this->idSites) . ")"; - $bind = $this->idSites; + $where = ""; + $bind = []; + if (!empty($this->idSites)) { + $where .= "$from.idsite IN (" . Common::getSqlStringFieldsArray($this->idSites) . ")"; + $bind = $this->idSites; + } if ($this->startDate instanceof Date) { $where .= " AND $from.$datetimeField >= ?"; $bind[] = $this->startDate->toString(Date::DATE_TIME_FORMAT); diff --git a/tests/PHPUnit/Integration/SegmentTest.php b/tests/PHPUnit/Integration/SegmentTest.php index 9c0e2464fea..399b094a42f 100644 --- a/tests/PHPUnit/Integration/SegmentTest.php +++ b/tests/PHPUnit/Integration/SegmentTest.php @@ -14,6 +14,7 @@ use Piwik\Common; use Piwik\Config; use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Db; use Piwik\Segment; use Piwik\Tests\Framework\Fixture; @@ -868,6 +869,99 @@ public function test_getSelectQuery_whenUnionOfSegmentsAreUsed() $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); } + public function test_getSelectQuery_whenUnionOfSegmentsAreUsedWithNotContainsCompare() + { + $select = 'log_visit.*'; + $from = 'log_visit'; + $where = false; + $bind = array(); + + $segment = 'actionUrl!@myTestUrl'; + $segment = new Segment($segment, $idSites = array()); + + $logVisitTable = Common::prefixTable('log_visit'); + $logLinkVisitActionTable = Common::prefixTable('log_link_visit_action'); + + $query = $segment->getSelectQuery($select, $from, $where, $bind); + + $expected = array( + "sql" => " SELECT log_visit.* FROM $logVisitTable AS log_visit + WHERE ( log_visit.idvisit NOT IN ( + SELECT log_inner.idvisit FROM ( + SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 3 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 2 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 10 )) ) ) + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) ", + "bind" => array('myTestUrl', 'myTestUrl', 'myTestUrl', 'myTestUrl')); + + $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); + } + + public function test_getSelectQuery_whenUsingNotEqualsCompareOnActionDimension() + { + $select = 'log_visit.*'; + $from = 'log_visit'; + $where = false; + $bind = array(); + + $segment = 'siteSearchCategory!=myCategory'; + $segment = new Segment($segment, $idSites = array()); + + $logVisitTable = Common::prefixTable('log_visit'); + $logLinkVisitActionTable = Common::prefixTable('log_link_visit_action'); + + $query = $segment->getSelectQuery($select, $from, $where, $bind); + + $expected = array( + "sql" => " SELECT log_visit.* FROM $logVisitTable AS log_visit + WHERE ( log_visit.idvisit NOT IN ( + SELECT log_inner.idvisit FROM ( + SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE log_link_visit_action.search_cat = ? + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) ", + "bind" => array('myCategory')); + + $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); + } + + public function test_getSelectQuery_whenUsingNotEqualsAndNotContainsCompareOnActionDimensionWithIdSitesAndDates() + { + $select = 'log_visit.*'; + $from = 'log_visit'; + $where = false; + $bind = array(); + + $segment = 'siteSearchCategory!=myCategory;actionUrl!@myTestUrl'; + $segment = new Segment($segment, $idSites = array(1,5), Date::factory('2020-02-02 12:00:00'), Date::factory('2020-02-05 09:00:00')); + + $logVisitTable = Common::prefixTable('log_visit'); + $logLinkVisitActionTable = Common::prefixTable('log_link_visit_action'); + + $query = $segment->getSelectQuery($select, $from, $where, $bind); + + $expected = array( + "sql" => " SELECT log_visit.* FROM $logVisitTable AS log_visit + WHERE ( log_visit.idvisit NOT IN ( + SELECT log_inner.idvisit FROM ( + SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_visit.idsite IN (?,?) AND log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? ) AND ( log_link_visit_action.search_cat = ? ) + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) AND + ( log_visit.idvisit NOT IN ( + SELECT log_inner.idvisit FROM ( + SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_visit.idsite IN (?,?) AND log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? ) AND + ( (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 3 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 2 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 10 )) ) )) + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) ", + "bind" => array(1, 5, '2020-02-02 12:00:00', '2020-02-05 09:00:00', 'myCategory', 1, 5, '2020-02-02 12:00:00', '2020-02-05 09:00:00', 'myTestUrl', 'myTestUrl', 'myTestUrl', 'myTestUrl')); + + $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); + } + public function test_getSelectQuery_whenJoinConversionOnLogLinkVisitAction_segmentUsesPageUrl() { $this->insertPageUrlAsAction('example.com/anypage'); From 92a915d8354f33c044d85d6b26c373dd985b560b Mon Sep 17 00:00:00 2001 From: sgiehl Date: Mon, 6 Jul 2020 13:49:49 +0200 Subject: [PATCH 10/18] adds system tests --- .../System/ManyVisitorsOneWebsiteTest.php | 17 + ...gment__Live.getLastVisitsDetails_month.xml | 1565 ++++++++++++++ ...gment__Live.getLastVisitsDetails_month.xml | 1817 +++++++++++++++++ 3 files changed, 3399 insertions(+) create mode 100644 tests/PHPUnit/System/expected/test_ManyVisitorsOneWebsiteTest_pageurlNotContainsSegment__Live.getLastVisitsDetails_month.xml create mode 100644 tests/PHPUnit/System/expected/test_ManyVisitorsOneWebsiteTest_siteSearchCategoryNotEqualsSegment__Live.getLastVisitsDetails_month.xml diff --git a/tests/PHPUnit/System/ManyVisitorsOneWebsiteTest.php b/tests/PHPUnit/System/ManyVisitorsOneWebsiteTest.php index 56dcdeee3f3..46e3cd24c08 100644 --- a/tests/PHPUnit/System/ManyVisitorsOneWebsiteTest.php +++ b/tests/PHPUnit/System/ManyVisitorsOneWebsiteTest.php @@ -148,6 +148,23 @@ public function getApiForTesting() 'otherRequestParameters' => array('filter_offset' => '4', 'filter_limit' => 3) )); + // #13785 + // check that not contains / not equals action segments filter away all visits having any matching action + $apiToTest[] = array('Live.getLastVisitsDetails', array( + 'idSite' => $idSite, + 'date' => $dateString, + 'periods' => 'month', + 'testSuffix' => '_pageurlNotContainsSegment', + 'segment' => 'pageUrl!@quest' + )); + $apiToTest[] = array('Live.getLastVisitsDetails', array( + 'idSite' => $idSite, + 'date' => $dateString, + 'periods' => 'month', + 'testSuffix' => '_siteSearchCategoryNotEqualsSegment', + 'segment' => 'siteSearchCategory!=CAT' + )); + // #8324 // testing filter_excludelowpop and filter_excludelowpop_value $apiToTest[] = array('UserCountry.getCountry', array( diff --git a/tests/PHPUnit/System/expected/test_ManyVisitorsOneWebsiteTest_pageurlNotContainsSegment__Live.getLastVisitsDetails_month.xml b/tests/PHPUnit/System/expected/test_ManyVisitorsOneWebsiteTest_pageurlNotContainsSegment__Live.getLastVisitsDetails_month.xml new file mode 100644 index 00000000000..01d74e66512 --- /dev/null +++ b/tests/PHPUnit/System/expected/test_ManyVisitorsOneWebsiteTest_pageurlNotContainsSegment__Live.getLastVisitsDetails_month.xml @@ -0,0 +1,1565 @@ + + + + 1 + 35 + 194.57.91.215 + + 5bc9628006479118 + + + action + http://piwik.net/grue/lair + It's pitch black... + 2 + + + 95 + + 1 + It's pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + goal + all + 1 + + 5 + 95 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + userid.email@example.org + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + search + Search Engines + Google + wikileaks ftw + + http://google.com/?q=Wikileaks FTW + http://google.com + plugins/Morpheus/icons/dist/searchEngines/google.com.png + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + WebKit + WebKit (Safari, Chrome) + Safari + Safari + plugins/Morpheus/icons/dist/browsers/SF.png + SF + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + North America + amn + United States + us + plugins/Morpheus/icons/dist/flags/us.png + California + CA + not a city + not a city, California, United States + 1 + 2 + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + + + 1 + 17 + 1.2.4.8 + + a36244db4114afa7 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 45 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 8 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 8 + + + + + goal + all + 1 + + 5 + 45 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Unknown + unk + Unknown + xx + plugins/Morpheus/icons/dist/flags/xx.png + + + + Unknown + + + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 8 + + + Cvar 5 name + Cvar5 value is 8 + + + + + 1 + 15 + 1.2.4.7 + + 28a7ee52024f3a89 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 40 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 7 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 7 + + + + + goal + all + 1 + + 5 + 40 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 3 + none + + 1 + 86400 + 0 + + 1 + 1s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + North Macedonia + mk + plugins/Morpheus/icons/dist/flags/mk.png + Gevgelija + 18 + Stratford-upon-Avon + Stratford-upon-Avon, Gevgelija, North Macedonia + + + 12:34:06 + 12 + 0 + 82800 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 7 + + + Cvar 5 name + Cvar5 value is 7 + + + + + 1 + 13 + 1.2.4.6 + + 289e2fcbb06929fa + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 34 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 6 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 6 + + + + + goal + all + 1 + + 5 + 34 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + Russia + ru + plugins/Morpheus/icons/dist/flags/ru.png + Sankt-Peterburg + SPE + Hluboká nad Vltavou + Hluboká nad Vltavou, Sankt-Peterburg, Russia + + + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 6 + + + Cvar 5 name + Cvar5 value is 6 + + + + + 1 + 11 + 1.2.4.5 + + 3e5f540b8952a4ab + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 29 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 5 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 5 + + + + + goal + all + 1 + + 5 + 29 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 3 + none + + 1 + 86400 + 0 + + 1 + 1s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + Russia + ru + plugins/Morpheus/icons/dist/flags/ru.png + Sankt-Peterburg + SPE + Stratford-upon-Avon + Stratford-upon-Avon, Sankt-Peterburg, Russia + + + 12:34:06 + 12 + 0 + 82800 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 5 + + + Cvar 5 name + Cvar5 value is 5 + + + + + 1 + 9 + 1.2.4.4 + + 17b5ac19cce8a192 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 23 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 4 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 4 + + + + + goal + all + 1 + + 5 + 23 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + United Kingdom + gb + plugins/Morpheus/icons/dist/flags/gb.png + Kent + KEN + Stratford-upon-Avon + Stratford-upon-Avon, Kent, United Kingdom + + + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 4 + + + Cvar 5 name + Cvar5 value is 4 + + + + + 1 + 7 + 1.2.4.3 + + 9b641f2d195745f4 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 18 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 3 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 3 + + + + + goal + all + 1 + + 5 + 18 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 3 + none + + 1 + 86400 + 0 + + 1 + 1s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + United Kingdom + gb + plugins/Morpheus/icons/dist/flags/gb.png + London, City of + LND + London + London, London, City of, United Kingdom + + + 12:34:06 + 12 + 0 + 82800 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 3 + + + Cvar 5 name + Cvar5 value is 3 + + + + + 1 + 29 + 113.62.1.1 + + d5a95c7fe2a8286d + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 79 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 3 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 3 + + + + + goal + all + 1 + + 5 + 79 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 3 + none + + 1 + 86400 + 0 + + 1 + 1s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + + Unknown + Tablet + plugins/Morpheus/icons/dist/devices/tablet.png + Sony + Xperia Tablet S + Android 4.1 + Android + plugins/Morpheus/icons/dist/os/AND.png + AND + 4.1 + Blink + Blink (Chrome, Opera) + Chrome 34.0 + Chrome + plugins/Morpheus/icons/dist/browsers/CH.png + CH + 34.0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Asia + asi + China + cn + plugins/Morpheus/icons/dist/flags/cn.png + Xizang Zizhiqu + XZ + Lhasa + Lhasa, Xizang Zizhiqu, China + 29.650000 + 91.100000 + 12:34:06 + 12 + 0 + 82800 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 3 + + + Cvar 5 name + Cvar5 value is 3 + + + + + 1 + 5 + 1.2.4.2 + + 85aaa85a2071daf5 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 12 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 2 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 2 + + + + + goal + all + 1 + + 5 + 12 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + United Kingdom + gb + plugins/Morpheus/icons/dist/flags/gb.png + Warwickshire + WAR + Stratford-upon-Avon + Stratford-upon-Avon, Warwickshire, United Kingdom + 124.456000 + 22.231000 + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 2 + + + Cvar 5 name + Cvar5 value is 2 + + + + + 1 + 27 + 2003:f6:93bf:26f:9ec7:a6ff:fe29:27df + + b6f1d5120b2b15a2 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 73 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 2 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 2 + + + + + goal + all + 1 + + 5 + 73 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + + Unknown + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Dell + Generic Desktop + Windows 7 + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + 7 + Trident + Trident (IE) + Internet Explorer 11.0 + Internet Explorer + plugins/Morpheus/icons/dist/browsers/IE.png + IE + 11.0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + North America + amn + United States + us + plugins/Morpheus/icons/dist/flags/us.png + + + + United States + 38 + -97 + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 2 + + + Cvar 5 name + Cvar5 value is 2 + + + + \ No newline at end of file diff --git a/tests/PHPUnit/System/expected/test_ManyVisitorsOneWebsiteTest_siteSearchCategoryNotEqualsSegment__Live.getLastVisitsDetails_month.xml b/tests/PHPUnit/System/expected/test_ManyVisitorsOneWebsiteTest_siteSearchCategoryNotEqualsSegment__Live.getLastVisitsDetails_month.xml new file mode 100644 index 00000000000..b376ceea5b4 --- /dev/null +++ b/tests/PHPUnit/System/expected/test_ManyVisitorsOneWebsiteTest_siteSearchCategoryNotEqualsSegment__Live.getLastVisitsDetails_month.xml @@ -0,0 +1,1817 @@ + + + + 1 + 35 + 194.57.91.215 + + 5bc9628006479118 + + + action + http://piwik.net/grue/lair + It's pitch black... + 2 + + + 95 + + 1 + It's pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + goal + all + 1 + + 5 + 95 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + userid.email@example.org + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + search + Search Engines + Google + wikileaks ftw + + http://google.com/?q=Wikileaks FTW + http://google.com + plugins/Morpheus/icons/dist/searchEngines/google.com.png + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + WebKit + WebKit (Safari, Chrome) + Safari + Safari + plugins/Morpheus/icons/dist/browsers/SF.png + SF + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + North America + amn + United States + us + plugins/Morpheus/icons/dist/flags/us.png + California + CA + not a city + not a city, California, United States + 1 + 2 + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + + + 1 + 17 + 1.2.4.8 + + a36244db4114afa7 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 45 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 8 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 8 + + + + + goal + all + 1 + + 5 + 45 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Unknown + unk + Unknown + xx + plugins/Morpheus/icons/dist/flags/xx.png + + + + Unknown + + + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 8 + + + Cvar 5 name + Cvar5 value is 8 + + + + + 1 + 16 + 1.2.4.7 + + 28a7ee52024f3a89 + + + action + http://piwik.net/space/quest/iv + Space Quest XII + 4 + + + 41 + + 180 + 3 min 0s + 1 + Space Quest XII + http://piwik.net/space/quest/iv + + plugins/Morpheus/images/action.svg + + 0 M + + + goal + all + 1 + + 5 + 41 + + http://piwik.net/space/quest/iv + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + goal + two + 2 + + 5 + + + http://piwik.net/space/quest/iv + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + two ($5 revenue) + + + + download + http://example.org/path/file7.zip + + 42 + + + 42 + + 180 + 3 min 0s + 1 + plugins/Morpheus/images/download.png + plugins/Morpheus/images/download.svg + Download + http://example.org/path/file7.zip + + 0 M + + + outlink + http://example-outlink.org/7.html + + 43 + + + 43 + + 180 + 3 min 0s + 1 + plugins/Morpheus/images/link.png + plugins/Morpheus/images/link.svg + Outlink + http://example-outlink.org/7.html + + 0 M + + + event + http://piwik.net/space/quest/iv + 8 + + + 44 + Cat7 + Action7 + + 1 + + plugins/Morpheus/images/event.png + plugins/Morpheus/images/event.svg + Event + Category: "Cat7', Action: "Action7" + Name7 + 352.678 + 0 M + + + 2 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 4 + none + + 1 + 90000 + 0 + + 1621 + 27 min 1s + 0 + 4 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + Europe + eur + North Macedonia + mk + plugins/Morpheus/icons/dist/flags/mk.png + Gevgelija + 18 + Stratford-upon-Avon + Stratford-upon-Avon, Gevgelija, North Macedonia + + + 12:34:06 + 12 + 0 + 3600 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 7 + + + Cvar 5 name + Cvar5 value is 7 + + + + + 1 + 15 + 1.2.4.7 + + 28a7ee52024f3a89 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 40 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 7 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 7 + + + + + goal + all + 1 + + 5 + 40 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 3 + none + + 1 + 86400 + 0 + + 1 + 1s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + North Macedonia + mk + plugins/Morpheus/icons/dist/flags/mk.png + Gevgelija + 18 + Stratford-upon-Avon + Stratford-upon-Avon, Gevgelija, North Macedonia + + + 12:34:06 + 12 + 0 + 82800 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 7 + + + Cvar 5 name + Cvar5 value is 7 + + + + + 1 + 13 + 1.2.4.6 + + 289e2fcbb06929fa + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 34 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 6 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 6 + + + + + goal + all + 1 + + 5 + 34 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + Russia + ru + plugins/Morpheus/icons/dist/flags/ru.png + Sankt-Peterburg + SPE + Hluboká nad Vltavou + Hluboká nad Vltavou, Sankt-Peterburg, Russia + + + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 6 + + + Cvar 5 name + Cvar5 value is 6 + + + + + 1 + 12 + 1.2.4.5 + + 3e5f540b8952a4ab + + + action + http://piwik.net/space/quest/iv + Space Quest XII + 4 + + + 30 + + 180 + 3 min 0s + 1 + Space Quest XII + http://piwik.net/space/quest/iv + + plugins/Morpheus/images/action.svg + + 0 M + + + goal + all + 1 + + 5 + 30 + + http://piwik.net/space/quest/iv + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + goal + two + 2 + + 5 + + + http://piwik.net/space/quest/iv + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + two ($5 revenue) + + + + download + http://example.org/path/file5.zip + + 32 + + + 31 + + 180 + 3 min 0s + 1 + plugins/Morpheus/images/download.png + plugins/Morpheus/images/download.svg + Download + http://example.org/path/file5.zip + + 0 M + + + outlink + http://example-outlink.org/5.html + + 33 + + + 32 + + 180 + 3 min 0s + 1 + plugins/Morpheus/images/link.png + plugins/Morpheus/images/link.svg + Outlink + http://example-outlink.org/5.html + + 0 M + + + event + http://piwik.net/space/quest/iv + 8 + + + 33 + Cat5 + Action5 + + 1 + + plugins/Morpheus/images/event.png + plugins/Morpheus/images/event.svg + Event + Category: "Cat5', Action: "Action5" + Name5 + 350.678 + 0 M + + + 2 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 4 + none + + 1 + 90000 + 0 + + 1621 + 27 min 1s + 0 + 4 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + Europe + eur + Russia + ru + plugins/Morpheus/icons/dist/flags/ru.png + Sankt-Peterburg + SPE + Stratford-upon-Avon + Stratford-upon-Avon, Sankt-Peterburg, Russia + + + 12:34:06 + 12 + 0 + 3600 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 5 + + + Cvar 5 name + Cvar5 value is 5 + + + + + 1 + 11 + 1.2.4.5 + + 3e5f540b8952a4ab + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 29 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 5 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 5 + + + + + goal + all + 1 + + 5 + 29 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 3 + none + + 1 + 86400 + 0 + + 1 + 1s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + Russia + ru + plugins/Morpheus/icons/dist/flags/ru.png + Sankt-Peterburg + SPE + Stratford-upon-Avon + Stratford-upon-Avon, Sankt-Peterburg, Russia + + + 12:34:06 + 12 + 0 + 82800 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 5 + + + Cvar 5 name + Cvar5 value is 5 + + + + + 1 + 9 + 1.2.4.4 + + 17b5ac19cce8a192 + + + action + http://piwik.net/grue/lair + It's <script> pitch black... + 2 + + + 23 + + 1 + It's <script> pitch black... + http://piwik.net/grue/lair + + plugins/Morpheus/images/action.svg + + 0 M + + + Cvar 2 PAGE name + Cvar2 PAGE value is 4 + + + Cvar 5 PAGE name + Cvar5 PAGE value is 4 + + + + + goal + all + 1 + + 5 + 23 + + http://piwik.net/grue/lair + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + 1 + USD + $ + + + + + Site 1 + + + + + + + + new + + 1 + plugins/Morpheus/images/goal.svg + 1 + none + + 0 + 0 + 0 + + 0 + 0s + 0 + 1 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + Europe + eur + United Kingdom + gb + plugins/Morpheus/icons/dist/flags/gb.png + Kent + KEN + Stratford-upon-Avon + Stratford-upon-Avon, Kent, United Kingdom + + + 12:34:06 + 12 + 0 + 0 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 4 + + + Cvar 5 name + Cvar5 value is 4 + + + + + 1 + 8 + 1.2.4.3 + + 9b641f2d195745f4 + + + action + http://piwik.net/space/quest/iv + Space Quest XII + 4 + + + 19 + + 180 + 3 min 0s + 1 + Space Quest XII + http://piwik.net/space/quest/iv + + plugins/Morpheus/images/action.svg + + 0 M + + + goal + all + 1 + + 5 + 19 + + http://piwik.net/space/quest/iv + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + goal + two + 2 + + 5 + + + http://piwik.net/space/quest/iv + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + two ($5 revenue) + + + + download + http://example.org/path/file3.zip + + 22 + + + 20 + + 180 + 3 min 0s + 1 + plugins/Morpheus/images/download.png + plugins/Morpheus/images/download.svg + Download + http://example.org/path/file3.zip + + 0 M + + + outlink + http://example-outlink.org/3.html + + 23 + + + 21 + + 180 + 3 min 0s + 1 + plugins/Morpheus/images/link.png + plugins/Morpheus/images/link.svg + Outlink + http://example-outlink.org/3.html + + 0 M + + + event + http://piwik.net/space/quest/iv + 8 + + + 22 + Cat3 + Action3 + + 1 + + plugins/Morpheus/images/event.png + plugins/Morpheus/images/event.svg + Event + Category: "Cat3', Action: "Action3" + Name3 + 348.678 + 0 M + + + 2 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 4 + none + + 1 + 90000 + 0 + + 1621 + 27 min 1s + 0 + 4 + 1 + direct + Direct Entry + + + + + + + + + fr + French + Desktop + plugins/Morpheus/icons/dist/devices/desktop.png + Unknown + Generic Desktop + Windows XP + Windows + plugins/Morpheus/icons/dist/os/WIN.png + WIN + XP + Gecko + Gecko (Firefox) + Firefox 3.6 + Firefox + plugins/Morpheus/icons/dist/browsers/FF.png + FF + 3.6 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + Europe + eur + United Kingdom + gb + plugins/Morpheus/icons/dist/flags/gb.png + London, City of + LND + London + London, London, City of, United Kingdom + + + 12:34:06 + 12 + 0 + 3600 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + Cvar 1 name + Cvar1 value is 3 + + + Cvar 5 name + Cvar5 value is 3 + + + + + 1 + 30 + 113.62.1.1 + + d5a95c7fe2a8286d + + + action + http://piwik.net/space/quest/iv + Space Quest XII + 4 + + + 80 + + 180 + 3 min 0s + 1 + Space Quest XII + http://piwik.net/space/quest/iv + + plugins/Morpheus/images/action.svg + + 0 M + + + goal + all + 1 + + 5 + 80 + + http://piwik.net/space/quest/iv + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + all ($5 revenue) + + + + goal + two + 2 + + 5 + + + http://piwik.net/space/quest/iv + plugins/Morpheus/images/goal.png + plugins/Morpheus/images/goal.svg + Goal conversion + two ($5 revenue) + + + + download + http://example.org/path/file3.zip + + 22 + + + 81 + + 180 + 3 min 0s + 1 + plugins/Morpheus/images/download.png + plugins/Morpheus/images/download.svg + Download + http://example.org/path/file3.zip + + 0 M + + + outlink + http://example-outlink.org/3.html + + 23 + + + 82 + + 180 + 3 min 0s + 1 + plugins/Morpheus/images/link.png + plugins/Morpheus/images/link.svg + Outlink + http://example-outlink.org/3.html + + 0 M + + + event + http://piwik.net/space/quest/iv + 8 + + + 83 + Cat3 + Action3 + + 1 + + plugins/Morpheus/images/event.png + plugins/Morpheus/images/event.svg + Event + Category: "Cat3', Action: "Action3" + Name3 + 348.678 + 0 M + + + 2 + USD + $ + + + + + Site 1 + + + + + + + + returning + plugins/Live/images/returningVisitor.png + 1 + plugins/Morpheus/images/goal.svg + 4 + none + + 1 + 90000 + 0 + + 1621 + 27 min 1s + 0 + 4 + 1 + direct + Direct Entry + + + + + + + + + + Unknown + Tablet + plugins/Morpheus/icons/dist/devices/tablet.png + Sony + Xperia Tablet S + Android 4.1 + Android + plugins/Morpheus/icons/dist/os/AND.png + AND + 4.1 + Blink + Blink (Chrome, Opera) + Chrome 34.0 + Chrome + plugins/Morpheus/icons/dist/browsers/CH.png + CH + 34.0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + Asia + asi + China + cn + plugins/Morpheus/icons/dist/flags/cn.png + Xizang Zizhiqu + XZ + Lhasa + Lhasa, Xizang Zizhiqu, China + 29.650000 + 91.100000 + 12:34:06 + 12 + 0 + 3600 + 1024x768 + cookie, flash, java + + + plugins/Morpheus/icons/dist/plugins/cookie.png + cookie + + + plugins/Morpheus/icons/dist/plugins/flash.png + flash + + + plugins/Morpheus/icons/dist/plugins/java.png + java + + + + + + \ No newline at end of file From f3e690677b4cafeb80bc57204b2a0ab65062a6f3 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Mon, 6 Jul 2020 16:33:24 +0200 Subject: [PATCH 11/18] update segment tests --- tests/PHPUnit/Integration/SegmentTest.php | 72 ++++++++++++++++------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/tests/PHPUnit/Integration/SegmentTest.php b/tests/PHPUnit/Integration/SegmentTest.php index 399b094a42f..6567d2f9eed 100644 --- a/tests/PHPUnit/Integration/SegmentTest.php +++ b/tests/PHPUnit/Integration/SegmentTest.php @@ -287,10 +287,16 @@ public function test_getSelectQuery_whenJoinActionOnConversion() FROM " . Common::prefixTable('log_conversion') . " AS log_conversion LEFT JOIN " . Common::prefixTable('log_link_visit_action') . " AS log_link_visit_action ON log_link_visit_action.idvisit = log_conversion.idvisit + LEFT JOIN log_visit AS log_visit ON log_visit.idvisit = log_conversion.idvisit WHERE ( log_conversion.idvisit = ? ) AND - ( ( log_conversion.idgoal IS NULL OR log_conversion.idgoal <> ? ) AND log_link_visit_action.search_cat = ? AND log_conversion.idgoal = ? )", + ( ( log_visit.idvisit NOT IN ( + SELECT log_inner.idvisit FROM ( + SELECT log_visit.idvisit FROM " . Common::prefixTable('log_visit') . " AS log_visit + LEFT JOIN " . Common::prefixTable('log_conversion') . " AS log_conversion ON log_conversion.idvisit = log_visit.idvisit + WHERE log_conversion.idgoal = ? GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) + AND log_link_visit_action.search_cat = ? AND log_conversion.idgoal = ? )", "bind" => array(1, 2, 'Test', 1)); $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); @@ -1404,12 +1410,18 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ LEFT JOIN " . Common::prefixTable('log_link_visit_action') . " AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit WHERE (HOUR(log_visit.visit_last_action_time) = ? OR (1 = 0)) " . // pageUrl==xyz - "AND ((1 = 1) " . // pageUrl!=abcdefg + "AND (( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit WHERE (1 = 0) ) ) " . // pageUrl!=abcdefg " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) " . // pageUrl=@does-not-exist - " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) )" . // pageUrl=@found-in-db - " OR log_link_visit_action.idaction_url = ?" . // pageUrl=='.urlencode($pageUrlFoundInDb) - " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name NOT LIKE CONCAT('%', ?, '%') AND type = 1 )) )" . // pageUrl!@not-found - " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name NOT LIKE CONCAT('%', ?, '%') AND type = 1 )) )" . // pageUrl!@found + " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) " . // pageUrl=@found-in-db + " OR log_link_visit_action.idaction_url = ? " . // pageUrl=='.urlencode($pageUrlFoundInDb) + " OR ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM + ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) " . // pageUrl!@not-found + " OR ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM + ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) )" . // pageUrl!@found " ) GROUP BY log_visit.idvisit ORDER BY NULL @@ -1467,7 +1479,7 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ WHERE (HOUR(log_visit.visit_last_action_time) = ? OR (1 = 0))" . // pageUrl==xyz " - AND ((1 = 1) " . // pageUrl!=abcdefg + AND (( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit WHERE (1 = 0) ) )" . // pageUrl!=abcdefg " OR (1 = 0) " . // pageUrl=@does-not-exist " @@ -1475,9 +1487,17 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ " OR log_link_visit_action.idaction_url = ?" . // pageUrl=='.urlencode($pageUrlFoundInDb) " - OR ( log_link_visit_action.idaction_url IN (?,?,?) )" . // pageUrl!@not-found + OR ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM ( + SELECT log_visit.idvisit FROM log_visit AS log_visit + LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_link_visit_action.idaction_url IN (?) ) + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) )" . // pageUrl!@not-found " - OR (1 = 0) " . // pageUrl!@found + OR ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM ( + SELECT log_visit.idvisit FROM log_visit AS log_visit + LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_link_visit_action.idaction_url IN (?,?,?,?) ) + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) " . // pageUrl!@found ") GROUP BY log_visit.idvisit ORDER BY NULL @@ -1488,9 +1508,11 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ 2, // pageUrl=@found-in-db 3, // pageUrl=@found-in-db $actionIdFoundInDb, // pageUrl=='.urlencode($pageUrlFoundInDb) - 1, // pageUrl!@not-found - 2, // pageUrl!@not-found - 3, // pageUrl!@not-found + 4, // pageUrl!@not-found + 1, // pageUrl!@found + 2, // pageUrl!@found + 3, // pageUrl!@found + 4, // pageUrl!@found )); $cache = StaticContainer::get('Piwik\Tracker\TableLogAction\Cache'); @@ -1505,13 +1527,13 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ $this->assertCacheWasHit($hits = 0); $this->test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_AND_withCacheSave(); - $this->assertCacheWasHit($hits = 8); + $this->assertCacheWasHit($hits = 20); $this->test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_AND_withCacheSave(); - $this->assertCacheWasHit($hits = 20); + $this->assertCacheWasHit($hits = 44); $this->test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_AND_withCacheSave(); - $this->assertCacheWasHit($hits = 32); + $this->assertCacheWasHit($hits = 68); } @@ -1531,9 +1553,9 @@ public function test_getSelectQuery_withTwoSegments_subqueryNotCached_whenResult /** * pageUrl=@found-in-db-bis -- Will be cached - * pageUrl!@not-found -- Too big to cache + * siteSearchCategory!@not-found -- Too big to cache */ - $segment = 'pageUrl=@found-in-db-bis;pageUrl!@not-found'; + $segment = 'pageUrl=@found-in-db-bis;siteSearchCategory!@not-found'; $segment = new Segment($segment, $idSites = array()); $query = $segment->getSelectQuery($select, $from, $where, $bind); @@ -1553,13 +1575,18 @@ public function test_getSelectQuery_withTwoSegments_subqueryNotCached_whenResult WHERE ( log_link_visit_action.idaction_url IN (?) )" . // pageUrl=@found-in-db-bis " - AND ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name NOT LIKE CONCAT('%', ?, '%') AND type = 1 )) ) " . // pageUrl!@not-found + AND ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM ( + SELECT log_visit.idvisit + FROM log_visit AS log_visit + LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE log_link_visit_action.search_cat LIKE ? + GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) " . // siteSearchCategory!@not-found "GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner", "bind" => array( 2, // pageUrl=@found-in-db-bis - "not-found", // pageUrl!@not-found + "%not-found%", // siteSearchCategory!@not-found )); $cache = StaticContainer::get('Piwik\Tracker\TableLogAction\Cache'); @@ -1575,11 +1602,11 @@ public function test_getSelectQuery_withTwoSegments_partiallyCached() // this will create the caches for both segments $this->test_getSelectQuery_withTwoSegments_subqueryNotCached_whenResultsetTooLarge(); - $this->assertCacheWasHit($hits = 4); + $this->assertCacheWasHit($hits = 2); // this will hit caches for both segments $this->test_getSelectQuery_withTwoSegments_subqueryNotCached_whenResultsetTooLarge(); - $this->assertCacheWasHit($hits = 10); + $this->assertCacheWasHit($hits = 5); } // se https://github.com/piwik/piwik/issues/9194 @@ -1709,7 +1736,7 @@ public function test_getSelectQuery_whenQueryingLogConversionWithSegmentThatUses SELECT log_inner.idgoal AS `idgoal`, count(*) AS `1`, count(distinct log_inner.idvisit) AS `3`, ROUND(SUM(log_inner.revenue),2) AS `2` FROM ( SELECT log_conversion.idgoal, log_conversion.idvisit, log_conversion.revenue - FROM log_conversion AS log_conversion + FROM $logConversionTable AS log_conversion LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_conversion.idvisit LEFT JOIN $logVisitTable AS log_visit ON log_visit.idvisit = log_conversion.idvisit WHERE ( log_conversion.server_time >= ? @@ -1752,6 +1779,7 @@ private function insertActions() // Adding some other actions to make test case more realistic $this->insertPageUrlAsAction('example.net/found-in-db-bis'); $this->insertPageUrlAsAction('example.net/found-in-db-ter'); + $this->insertPageUrlAsAction('example.net/page-not-found'); return array($pageUrlFoundInDb, $actionIdFoundInDb); } From 2fd13ad4fb21bc790e2129a38eeea8319af91c54 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Tue, 7 Jul 2020 10:05:40 +0200 Subject: [PATCH 12/18] avoid additional subselect --- core/Segment.php | 5 ++ tests/PHPUnit/Integration/SegmentTest.php | 81 ++++++++++------------- 2 files changed, 40 insertions(+), 46 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index 354762acabe..f5a1855b293 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -381,7 +381,12 @@ protected function getCleanedExpression($expression) $where .= " AND $from.$datetimeField <= ?"; $bind[] = $this->endDate->toString(Date::DATE_TIME_FORMAT); } + + $logQueryBuilder = StaticContainer::get('Piwik\DataAccess\LogQueryBuilder'); + $forceGroupByBackup = $logQueryBuilder->getForcedInnerGroupBySubselect(); + $logQueryBuilder->forceInnerGroupBySubselect(LogQueryBuilder::FORCE_INNER_GROUP_BY_NO_SUBSELECT); $query = $segmentObj->getSelectQuery($select, $from, $where, $bind); + $logQueryBuilder->forceInnerGroupBySubselect($forceGroupByBackup); return ['log_visit.idvisit', SegmentExpression::MATCH_ACTIONS_NOT_CONTAINS, $query]; } diff --git a/tests/PHPUnit/Integration/SegmentTest.php b/tests/PHPUnit/Integration/SegmentTest.php index 6567d2f9eed..9701563580e 100644 --- a/tests/PHPUnit/Integration/SegmentTest.php +++ b/tests/PHPUnit/Integration/SegmentTest.php @@ -10,7 +10,6 @@ use Exception; use Piwik\ArchiveProcessor\Rules; -use Piwik\Cache; use Piwik\Common; use Piwik\Config; use Piwik\Container\StaticContainer; @@ -292,10 +291,9 @@ public function test_getSelectQuery_whenJoinActionOnConversion() ( log_conversion.idvisit = ? ) AND ( ( log_visit.idvisit NOT IN ( - SELECT log_inner.idvisit FROM ( - SELECT log_visit.idvisit FROM " . Common::prefixTable('log_visit') . " AS log_visit - LEFT JOIN " . Common::prefixTable('log_conversion') . " AS log_conversion ON log_conversion.idvisit = log_visit.idvisit - WHERE log_conversion.idgoal = ? GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) + SELECT log_visit.idvisit FROM " . Common::prefixTable('log_visit') . " AS log_visit + LEFT JOIN " . Common::prefixTable('log_conversion') . " AS log_conversion ON log_conversion.idvisit = log_visit.idvisit + WHERE log_conversion.idgoal = ? ) ) AND log_link_visit_action.search_cat = ? AND log_conversion.idgoal = ? )", "bind" => array(1, 2, 'Test', 1)); @@ -892,15 +890,13 @@ public function test_getSelectQuery_whenUnionOfSegmentsAreUsedWithNotContainsCom $expected = array( "sql" => " SELECT log_visit.* FROM $logVisitTable AS log_visit - WHERE ( log_visit.idvisit NOT IN ( - SELECT log_inner.idvisit FROM ( - SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR - ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 3 )) ) OR - ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 2 )) ) OR - ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 10 )) ) ) - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) ", - "bind" => array('myTestUrl', 'myTestUrl', 'myTestUrl', 'myTestUrl')); + WHERE ( log_visit.idvisit NOT IN ( + SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 3 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 2 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 10 )) ) )) ) ", + "bind" => array('myTestUrl', 'myTestUrl', 'myTestUrl', 'myTestUrl')); $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); } @@ -922,11 +918,9 @@ public function test_getSelectQuery_whenUsingNotEqualsCompareOnActionDimension() $expected = array( "sql" => " SELECT log_visit.* FROM $logVisitTable AS log_visit - WHERE ( log_visit.idvisit NOT IN ( - SELECT log_inner.idvisit FROM ( - SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE log_link_visit_action.search_cat = ? - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) ", + WHERE ( log_visit.idvisit NOT IN ( + SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE log_link_visit_action.search_cat = ? ) ) ", "bind" => array('myCategory')); $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); @@ -949,20 +943,16 @@ public function test_getSelectQuery_whenUsingNotEqualsAndNotContainsCompareOnAct $expected = array( "sql" => " SELECT log_visit.* FROM $logVisitTable AS log_visit - WHERE ( log_visit.idvisit NOT IN ( - SELECT log_inner.idvisit FROM ( - SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE ( log_visit.idsite IN (?,?) AND log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? ) AND ( log_link_visit_action.search_cat = ? ) - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) AND - ( log_visit.idvisit NOT IN ( - SELECT log_inner.idvisit FROM ( - SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE ( log_visit.idsite IN (?,?) AND log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? ) AND - ( (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR - ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 3 )) ) OR - ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 2 )) ) OR - ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 10 )) ) )) - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) ", + WHERE ( log_visit.idvisit NOT IN ( + SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_visit.idsite IN (?,?) AND log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? ) AND ( log_link_visit_action.search_cat = ? )) ) + AND ( log_visit.idvisit NOT IN ( + SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_visit.idsite IN (?,?) AND log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? ) AND + ( (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 3 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 2 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 10 )) ) ))) ) ", "bind" => array(1, 5, '2020-02-02 12:00:00', '2020-02-05 09:00:00', 'myCategory', 1, 5, '2020-02-02 12:00:00', '2020-02-05 09:00:00', 'myTestUrl', 'myTestUrl', 'myTestUrl', 'myTestUrl')); $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); @@ -1414,14 +1404,14 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) " . // pageUrl=@does-not-exist " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) " . // pageUrl=@found-in-db " OR log_link_visit_action.idaction_url = ? " . // pageUrl=='.urlencode($pageUrlFoundInDb) - " OR ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM - ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + " OR ( log_visit.idvisit NOT IN ( + SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit WHERE ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) " . // pageUrl!@not-found - " OR ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM - ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + ) ) " . // pageUrl!@not-found + " OR ( log_visit.idvisit NOT IN ( + SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit WHERE ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) )" . // pageUrl!@found + ) )" . // pageUrl!@found " ) GROUP BY log_visit.idvisit ORDER BY NULL @@ -1487,17 +1477,17 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ " OR log_link_visit_action.idaction_url = ?" . // pageUrl=='.urlencode($pageUrlFoundInDb) " - OR ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM ( + OR ( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit WHERE ( log_link_visit_action.idaction_url IN (?) ) - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) )" . // pageUrl!@not-found + ) )" . // pageUrl!@not-found " - OR ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM ( + OR ( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit WHERE ( log_link_visit_action.idaction_url IN (?,?,?,?) ) - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) " . // pageUrl!@found + ) ) " . // pageUrl!@found ") GROUP BY log_visit.idvisit ORDER BY NULL @@ -1575,12 +1565,11 @@ public function test_getSelectQuery_withTwoSegments_subqueryNotCached_whenResult WHERE ( log_link_visit_action.idaction_url IN (?) )" . // pageUrl=@found-in-db-bis " - AND ( log_visit.idvisit NOT IN ( SELECT log_inner.idvisit FROM ( + AND ( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE log_link_visit_action.search_cat LIKE ? - GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner) ) " . // siteSearchCategory!@not-found + WHERE log_link_visit_action.search_cat LIKE ? ) ) " . // siteSearchCategory!@not-found "GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner", From 0c87fe746df8d345bf75ede5a9cd755c451dcb00 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Tue, 7 Jul 2020 10:28:24 +0200 Subject: [PATCH 13/18] submodule changes --- plugins/CustomVariables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/CustomVariables b/plugins/CustomVariables index 9b96b7eff45..49a252c8579 160000 --- a/plugins/CustomVariables +++ b/plugins/CustomVariables @@ -1 +1 @@ -Subproject commit 9b96b7eff455fe63fd2247dbc6a18416905d0dac +Subproject commit 49a252c85792d4d6f52acb1232004deb180790d5 From 66f38f23bbdaa5decef95879dce49f775a75c2b2 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Wed, 8 Jul 2020 09:08:52 +0200 Subject: [PATCH 14/18] adjust ArchiveQueryFactory segment initialize --- core/Archive/ArchiveQueryFactory.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/Archive/ArchiveQueryFactory.php b/core/Archive/ArchiveQueryFactory.php index de69f39066b..9359f0c584c 100644 --- a/core/Archive/ArchiveQueryFactory.php +++ b/core/Archive/ArchiveQueryFactory.php @@ -29,7 +29,7 @@ public function build($idSites, $strPeriod, $strDate, $strSegment = false, $_res { list($websiteIds, $timezone, $idSiteIsAll) = $this->getSiteInfoFromQueryParam($idSites, $_restrictSitesToLogin); list($allPeriods, $isMultipleDate) = $this->getPeriodInfoFromQueryParam($strDate, $strPeriod, $timezone); - $segment = $this->getSegmentFromQueryParam($strSegment, $websiteIds); + $segment = $this->getSegmentFromQueryParam($strSegment, $websiteIds, $allPeriods); return $this->factory($segment, $allPeriods, $websiteIds, $idSiteIsAll, $isMultipleDate); } @@ -118,10 +118,13 @@ protected function getPeriodInfoFromQueryParam($strDate, $strPeriod, $timezone) * * @param string $strSegment the value of the 'segment' query parameter. * @param int[] $websiteIds the list of sites being queried. + * @param Period[] $allPeriods list of all periods * @return Segment */ - protected function getSegmentFromQueryParam($strSegment, $websiteIds) + protected function getSegmentFromQueryParam($strSegment, $websiteIds, $allPeriods) { - return new Segment($strSegment, $websiteIds); + // we might have multiple periods, so use the start date of the first one and + // the end date of the last one to limit the possible segment subquery + return new Segment($strSegment, $websiteIds, reset($allPeriods)->getDateTimeStart(), end($allPeriods)->getDateTimeEnd()); } } \ No newline at end of file From ea1bafd9fc63f451cdbc0177a2e43ac70a80f023 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Thu, 9 Jul 2020 10:46:04 +0200 Subject: [PATCH 15/18] require a start date for segment subqueries --- core/Segment.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index f5a1855b293..0b8e786757d 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -298,10 +298,18 @@ private function isVisitSegment($name) private function doesSegmentNeedSubquery($operator, $segmentName) { - return in_array($operator, [ + $requiresSubQuery = in_array($operator, [ SegmentExpression::MATCH_DOES_NOT_CONTAIN, SegmentExpression::MATCH_NOT_EQUAL ]) && !$this->isVisitSegment($segmentName); + + if ($requiresSubQuery && empty($this->startDate)) { + $e = new Exception(); + Log::warning("Avoiding segment subquery due to missing start date. Please ensure a start date is set when initializing a segment if it's used to build a query. Stacktrace:\n" . $e->getTraceAsString()); + return false; + } + + return $requiresSubQuery; } private function getInvertedOperatorForSubQuery($operator) @@ -362,7 +370,7 @@ protected function getCleanedExpression($expression) if ($this->doesSegmentNeedSubquery($matchType, $name)) { $operator = $this->getInvertedOperatorForSubQuery($matchType); $stringSegment = $name . $operator . $value; - $segmentObj = new Segment($stringSegment, $this->idSites); + $segmentObj = new Segment($stringSegment, $this->idSites, $this->startDate, $this->endDate); $select = 'log_visit.idvisit'; $from = 'log_visit'; From 3d02e8af98db697777e4ec18654f94839f71ea23 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Thu, 9 Jul 2020 11:32:50 +0200 Subject: [PATCH 16/18] fix where condition --- core/Segment.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index 0b8e786757d..08a720e745f 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -375,25 +375,25 @@ protected function getCleanedExpression($expression) $select = 'log_visit.idvisit'; $from = 'log_visit'; $datetimeField = 'visit_last_action_time'; - $where = ""; + $where = []; $bind = []; if (!empty($this->idSites)) { - $where .= "$from.idsite IN (" . Common::getSqlStringFieldsArray($this->idSites) . ")"; + $where[] = "$from.idsite IN (" . Common::getSqlStringFieldsArray($this->idSites) . ")"; $bind = $this->idSites; } if ($this->startDate instanceof Date) { - $where .= " AND $from.$datetimeField >= ?"; + $where[] = "$from.$datetimeField >= ?"; $bind[] = $this->startDate->toString(Date::DATE_TIME_FORMAT); } if ($this->endDate instanceof Date) { - $where .= " AND $from.$datetimeField <= ?"; + $where[] = "$from.$datetimeField <= ?"; $bind[] = $this->endDate->toString(Date::DATE_TIME_FORMAT); } $logQueryBuilder = StaticContainer::get('Piwik\DataAccess\LogQueryBuilder'); $forceGroupByBackup = $logQueryBuilder->getForcedInnerGroupBySubselect(); $logQueryBuilder->forceInnerGroupBySubselect(LogQueryBuilder::FORCE_INNER_GROUP_BY_NO_SUBSELECT); - $query = $segmentObj->getSelectQuery($select, $from, $where, $bind); + $query = $segmentObj->getSelectQuery($select, $from, implode(' AND ', $where), $bind); $logQueryBuilder->forceInnerGroupBySubselect($forceGroupByBackup); return ['log_visit.idvisit', SegmentExpression::MATCH_ACTIONS_NOT_CONTAINS, $query]; From e991fb285d468f4eb2fe39f29422f359fce80397 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Thu, 9 Jul 2020 16:36:57 +0200 Subject: [PATCH 17/18] update/add test --- tests/PHPUnit/Integration/SegmentTest.php | 96 ++++++++++++++++------- 1 file changed, 68 insertions(+), 28 deletions(-) diff --git a/tests/PHPUnit/Integration/SegmentTest.php b/tests/PHPUnit/Integration/SegmentTest.php index 9701563580e..ecca62eaf3c 100644 --- a/tests/PHPUnit/Integration/SegmentTest.php +++ b/tests/PHPUnit/Integration/SegmentTest.php @@ -274,7 +274,7 @@ public function test_getSelectQuery_whenJoinActionOnConversion() $bind = array(1); $segment = 'visitConvertedGoalId!=2;siteSearchCategory==Test;visitConvertedGoalId==1'; - $segment = new Segment($segment, $idSites = array()); + $segment = new Segment($segment, $idSites = array(), Date::factory('2020-02-02 02:00:00')); $query = $segment->getSelectQuery($select, $from, $where, $bind); $this->assertQueryDoesNotFail($query); @@ -293,9 +293,9 @@ public function test_getSelectQuery_whenJoinActionOnConversion() ( ( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM " . Common::prefixTable('log_visit') . " AS log_visit LEFT JOIN " . Common::prefixTable('log_conversion') . " AS log_conversion ON log_conversion.idvisit = log_visit.idvisit - WHERE log_conversion.idgoal = ? ) ) + WHERE ( log_visit.visit_last_action_time >= ? ) AND ( log_conversion.idgoal = ? )) ) AND log_link_visit_action.search_cat = ? AND log_conversion.idgoal = ? )", - "bind" => array(1, 2, 'Test', 1)); + "bind" => array(1, '2020-02-02 02:00:00', 2, 'Test', 1)); $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); } @@ -873,7 +873,7 @@ public function test_getSelectQuery_whenUnionOfSegmentsAreUsed() $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); } - public function test_getSelectQuery_whenUnionOfSegmentsAreUsedWithNotContainsCompare() + public function test_getSelectQuery_whenUnionOfSegmentsAreUsedWithNotContainsCompare_usesSubQueryWithGivenStartDate() { $select = 'log_visit.*'; $from = 'log_visit'; @@ -881,7 +881,7 @@ public function test_getSelectQuery_whenUnionOfSegmentsAreUsedWithNotContainsCom $bind = array(); $segment = 'actionUrl!@myTestUrl'; - $segment = new Segment($segment, $idSites = array()); + $segment = new Segment($segment, $idSites = array(), Date::factory('2020-02-02 02:00:00'), Date::factory('2020-02-29 02:00:00')); $logVisitTable = Common::prefixTable('log_visit'); $logLinkVisitActionTable = Common::prefixTable('log_link_visit_action'); @@ -892,10 +892,43 @@ public function test_getSelectQuery_whenUnionOfSegmentsAreUsedWithNotContainsCom "sql" => " SELECT log_visit.* FROM $logVisitTable AS log_visit WHERE ( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR + WHERE ( log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? ) AND ( (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 3 )) ) OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 2 )) ) OR - ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 10 )) ) )) ) ", + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 10 )) ) ))) ) ", + "bind" => array('2020-02-02 02:00:00', '2020-02-29 02:00:00', 'myTestUrl', 'myTestUrl', 'myTestUrl', 'myTestUrl')); + + $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); + } + + public function test_getSelectQuery_whenUnionOfSegmentsAreUsedWithNotContainsCompare_usesNoSubQueryWithoutStartDate() + { + $select = 'log_visit.*'; + $from = 'log_visit'; + $where = false; + $bind = array(); + + $segment = 'actionUrl!@myTestUrl'; + + // When no start date is given for the segment object, it will not generate a subquery, as it might have too many results + // instead it will try to directly join the tables, which might cause incorrect results for action dimensions + $segment = new Segment($segment, $idSites = array(), $startDate = null, $endDate = null); + + $logVisitTable = Common::prefixTable('log_visit'); + $logLinkVisitActionTable = Common::prefixTable('log_link_visit_action'); + + $query = $segment->getSelectQuery($select, $from, $where, $bind); + + $expected = array( + "sql" => " SELECT log_inner.* FROM ( + SELECT log_visit.* FROM $logVisitTable AS log_visit + LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE (( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name NOT LIKE CONCAT('%', ?, '%') AND type = 1 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name NOT LIKE CONCAT('%', ?, '%') AND type = 3 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name NOT LIKE CONCAT('%', ?, '%') AND type = 2 )) ) OR + ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name NOT LIKE CONCAT('%', ?, '%') AND type = 10 )) ) ) + GROUP BY log_visit.idvisit ORDER BY NULL ) + AS log_inner", "bind" => array('myTestUrl', 'myTestUrl', 'myTestUrl', 'myTestUrl')); $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); @@ -909,7 +942,7 @@ public function test_getSelectQuery_whenUsingNotEqualsCompareOnActionDimension() $bind = array(); $segment = 'siteSearchCategory!=myCategory'; - $segment = new Segment($segment, $idSites = array()); + $segment = new Segment($segment, $idSites = array(), Date::factory('2020-02-02 02:00:00')); $logVisitTable = Common::prefixTable('log_visit'); $logLinkVisitActionTable = Common::prefixTable('log_link_visit_action'); @@ -920,8 +953,8 @@ public function test_getSelectQuery_whenUsingNotEqualsCompareOnActionDimension() "sql" => " SELECT log_visit.* FROM $logVisitTable AS log_visit WHERE ( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM $logVisitTable AS log_visit LEFT JOIN $logLinkVisitActionTable AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE log_link_visit_action.search_cat = ? ) ) ", - "bind" => array('myCategory')); + WHERE ( log_visit.visit_last_action_time >= ? ) AND ( log_link_visit_action.search_cat = ? )) ) ", + "bind" => array('2020-02-02 02:00:00', 'myCategory')); $this->assertEquals($this->removeExtraWhiteSpaces($expected), $this->removeExtraWhiteSpaces($query)); } @@ -1383,7 +1416,7 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ * pageUrl!@found -- Matches none */ $segment = 'visitServerHour==12,pageUrl==xyz;pageUrl!=abcdefg,pageUrl=@does-not-exist,pageUrl=@found-in-db,pageUrl=='.urlencode($pageUrlFoundInDb).',pageUrl!@not-found,pageUrl!@found'; - $segment = new Segment($segment, $idSites = array()); + $segment = new Segment($segment, $idSites = array(), Date::factory('2020-02-02 02:00:00')); $query = $segment->getSelectQuery($select, $from, $where, $bind); @@ -1400,28 +1433,31 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ LEFT JOIN " . Common::prefixTable('log_link_visit_action') . " AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit WHERE (HOUR(log_visit.visit_last_action_time) = ? OR (1 = 0)) " . // pageUrl==xyz - "AND (( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit WHERE (1 = 0) ) ) " . // pageUrl!=abcdefg + "AND (( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit WHERE ( log_visit.visit_last_action_time >= ? ) AND ( (1 = 0) )) ) " . // pageUrl!=abcdefg " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) " . // pageUrl=@does-not-exist " OR ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) " . // pageUrl=@found-in-db " OR log_link_visit_action.idaction_url = ? " . // pageUrl=='.urlencode($pageUrlFoundInDb) " OR ( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) - ) ) " . // pageUrl!@not-found + WHERE ( log_visit.visit_last_action_time >= ? ) AND ( ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) + )) ) " . // pageUrl!@not-found " OR ( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) - ) )" . // pageUrl!@found + WHERE ( log_visit.visit_last_action_time >= ? ) AND ( ( log_link_visit_action.idaction_url IN (SELECT idaction FROM log_action WHERE ( name LIKE CONCAT('%', ?, '%') AND type = 1 )) ) + )) )" . // pageUrl!@found " ) GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner", "bind" => array( 12, + '2020-02-02 02:00:00', "does-not-exist", "found-in-db", $actionIdFoundInDb, + '2020-02-02 02:00:00', "not-found", + '2020-02-02 02:00:00', "found", )); @@ -1451,7 +1487,7 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ * pageUrl!@found -- Matches none */ $segment = 'visitServerHour==12,pageUrl==xyz;pageUrl!=abcdefg,pageUrl=@does-not-exist,pageUrl=@found-in-db,pageUrl=='.urlencode($pageUrlFoundInDb).',pageUrl!@not-found,pageUrl!@found'; - $segment = new Segment($segment, $idSites = array()); + $segment = new Segment($segment, $idSites = array(), Date::factory('2020-02-02 02:00:00')); $query = $segment->getSelectQuery($select, $from, $where, $bind); @@ -1469,7 +1505,7 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ WHERE (HOUR(log_visit.visit_last_action_time) = ? OR (1 = 0))" . // pageUrl==xyz " - AND (( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM log_visit AS log_visit WHERE (1 = 0) ) )" . // pageUrl!=abcdefg + AND (( log_visit.idvisit NOT IN ( SELECT log_visit.idvisit FROM " . Common::prefixTable('log_visit') . " AS log_visit WHERE ( log_visit.visit_last_action_time >= ? ) AND ( (1 = 0) )) )" . // pageUrl!=abcdefg " OR (1 = 0) " . // pageUrl=@does-not-exist " @@ -1478,27 +1514,30 @@ public function test_getSelectQuery_whenPageUrlDoesNotExist_asBothStatements_OR_ OR log_link_visit_action.idaction_url = ?" . // pageUrl=='.urlencode($pageUrlFoundInDb) " OR ( log_visit.idvisit NOT IN ( - SELECT log_visit.idvisit FROM log_visit AS log_visit - LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE ( log_link_visit_action.idaction_url IN (?) ) - ) )" . // pageUrl!@not-found + SELECT log_visit.idvisit FROM " . Common::prefixTable('log_visit') . " AS log_visit + LEFT JOIN " . Common::prefixTable('log_link_visit_action') . " AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_visit.visit_last_action_time >= ? ) AND ( ( log_link_visit_action.idaction_url IN (?) ) + )) )" . // pageUrl!@not-found " OR ( log_visit.idvisit NOT IN ( - SELECT log_visit.idvisit FROM log_visit AS log_visit - LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE ( log_link_visit_action.idaction_url IN (?,?,?,?) ) - ) ) " . // pageUrl!@found + SELECT log_visit.idvisit FROM " . Common::prefixTable('log_visit') . " AS log_visit + LEFT JOIN " . Common::prefixTable('log_link_visit_action') . " AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit + WHERE ( log_visit.visit_last_action_time >= ? ) AND ( ( log_link_visit_action.idaction_url IN (?,?,?,?) ) + )) ) " . // pageUrl!@found ") GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner", "bind" => array( 12, + '2020-02-02 02:00:00', 1, // pageUrl=@found-in-db 2, // pageUrl=@found-in-db 3, // pageUrl=@found-in-db $actionIdFoundInDb, // pageUrl=='.urlencode($pageUrlFoundInDb) + '2020-02-02 02:00:00', 4, // pageUrl!@not-found + '2020-02-02 02:00:00', 1, // pageUrl!@found 2, // pageUrl!@found 3, // pageUrl!@found @@ -1546,7 +1585,7 @@ public function test_getSelectQuery_withTwoSegments_subqueryNotCached_whenResult * siteSearchCategory!@not-found -- Too big to cache */ $segment = 'pageUrl=@found-in-db-bis;siteSearchCategory!@not-found'; - $segment = new Segment($segment, $idSites = array()); + $segment = new Segment($segment, $idSites = array(), Date::factory('2020-02-02 02:00:00')); $query = $segment->getSelectQuery($select, $from, $where, $bind); $this->assertQueryDoesNotFail($query); @@ -1569,12 +1608,13 @@ public function test_getSelectQuery_withTwoSegments_subqueryNotCached_whenResult SELECT log_visit.idvisit FROM log_visit AS log_visit LEFT JOIN log_link_visit_action AS log_link_visit_action ON log_link_visit_action.idvisit = log_visit.idvisit - WHERE log_link_visit_action.search_cat LIKE ? ) ) " . // siteSearchCategory!@not-found + WHERE ( log_visit.visit_last_action_time >= ? ) AND ( log_link_visit_action.search_cat LIKE ? )) ) " . // siteSearchCategory!@not-found "GROUP BY log_visit.idvisit ORDER BY NULL ) AS log_inner", "bind" => array( 2, // pageUrl=@found-in-db-bis + '2020-02-02 02:00:00', "%not-found%", // siteSearchCategory!@not-found )); From dceda1553158fa52573efd1981f525fa9e01d491 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Thu, 9 Jul 2020 16:44:30 +0200 Subject: [PATCH 18/18] add a note about startdate requirement --- core/Segment.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/Segment.php b/core/Segment.php index 08a720e745f..7062f102cda 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -103,11 +103,17 @@ class Segment /** * Constructor. * + * When using segments that contain a != or !@ condition on a non visit dimension (e.g. action, conversion, ...) it + * is needed to use a subquery to get correct results. To avoid subqueries that fetch too many data it's required to + * set a startDate in this case. That date will be used to limit the subquery (along with possibly given idSites or + * endDate). If no startDate is given for such a segment it will generate a query that directly joins the according + * tables, but trigger a php warning as results might be incorrect. + * * @param string $segmentCondition The segment condition, eg, `'browserCode=ff;countryCode=CA'`. * @param array $idSites The list of sites the segment will be used with. Some segments are * dependent on the site, such as goal segments. - * @param Date|null $startDate - * @param Date|null $endDate + * @param Date|null $startDate start date used to limit subqueries + * @param Date|null $endDate end date used to limit subqueries * @throws */ public function __construct($segmentCondition, $idSites, Date $startDate = null, Date $endDate = null)