Fix for HTML entity decoding of single quotes in campaigns and csv/xm…

…l exports (#18170)

* Tweaked html entity decoding to include single quotes for visitor detail campaign and csv/xml exports

* Decode HTML entities for outlinks on the visits log

* Updated UI tests for the live visitor log with quote special characters

* Update UI test screenshots

* Update UI test screenshots

* update submodule

Co-authored-by: sgiehl <[email protected]>

core/DataTable/Renderer.php

@@ -184,7 +184,7 @@ public static function formatValueXml($value)
         if (is_string($value)
             && !is_numeric($value)
         ) {
-            $value = html_entity_decode($value, ENT_COMPAT, 'UTF-8');
+            $value = html_entity_decode($value, ENT_QUOTES, 'UTF-8');
             // make sure non-UTF-8 chars don't cause htmlspecialchars to choke
             if (function_exists('mb_convert_encoding')) {
                 $value = @mb_convert_encoding($value, 'UTF-8', 'UTF-8');

core/DataTable/Renderer/Csv.php

@@ -241,7 +241,7 @@ protected function formatValue($value)
         if (is_string($value)
             && !is_numeric($value)
         ) {
-            $value = html_entity_decode($value, ENT_COMPAT, 'UTF-8');
+            $value = html_entity_decode($value, ENT_QUOTES, 'UTF-8');
         } elseif ($value === false) {
             $value = 0;
         }

plugins/Live/VisitorDetails.php

@@ -95,6 +95,10 @@ public function renderAction($action, $previousAction, $visitorDetails)
 
         $sitesModel = new \Piwik\Plugins\SitesManager\Model();
 
+        if (isset($action['type']) && $action['type'] == 'outlink' && isset($action['url'])) {
+            $action['url'] = html_entity_decode($action['url'], ENT_QUOTES, "UTF-8");
+        }
+
         $view                 = new View($template);
         $view->sendHeadersWhenRendering = false;
         $view->mainUrl        = trim(Site::getMainUrlFor($this->getIdSite()));

plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php

@@ -19,6 +19,7 @@ class VisitsWithAllActionsAndDevices extends Fixture
 {
     public $dateTime = '2010-02-01 11:22:33';
     public $idSite = 1;
+    private $specialChars = "sc_'_".'"'.'_%27_%22';
 
     public function setUp(): void
     {
@@ -31,14 +32,21 @@ public function setUp(): void
         CustomDimensionsApi::getInstance()->configureNewCustomDimension(1, 'age', 'visit', 1);
         CustomDimensionsApi::getInstance()->configureNewCustomDimension(1, 'currency', 'action', 1);
 
+        // Campaign - needs a separate user otherwise it overrides the referrer
         $t = self::getTracker($this->idSite, $this->dateTime, $defaultInit = true);
         $t->setTokenAuth(self::getTokenAuth());
+        $t->setUserId('Z4F66G776HGI');
+        $t->setVisitorId(substr(sha1('Z4F66G776HGI'), 0, 16));
+        $this->trackCampaignVisit($t, Date::factory($this->dateTime)->addHour(0)->getDatetime());
 
+        // Main user for all other visits
+        $t = self::getTracker($this->idSite, $this->dateTime, $defaultInit = true);
+        $t->setTokenAuth(self::getTokenAuth());
         $t->setUserId('X4F66G776HGI');
         $t->setVisitorId(substr(sha1('X4F66G776HGI'), 0, 16));
 
         // smart display
-        $this->trackDeviceVisit($t, Date::factory($this->dateTime)->addHour(0)->getDatetime(), 'Mozilla/5.0 (Linux; U; Android 4.0.4; de-de; VSD220 Build/IMM76D.UI23ED12_VSC) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30');
+        $this->trackDeviceVisit($t, Date::factory($this->dateTime)->addHour(1)->getDatetime(), 'Mozilla/5.0 (Linux; U; Android 4.0.4; de-de; VSD220 Build/IMM76D.UI23ED12_VSC) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30');
 
         // media player
         $this->trackVisitMediaPlayer($t, Date::factory($this->dateTime)->addHour(6.33)->getDatetime());
@@ -144,21 +152,22 @@ private function trackVisitSmartphone(\MatomoTracker $t, $dateTime)
         self::checkResponse($t->doTrackPageView('cart'));
 
         $t->setForceVisitDateTime(Date::factory($dateTime)->addHour(0.5)->getDatetime());
-        self::checkResponse($t->doTrackAction('http://vendor.site', 'link'));
+        self::checkResponse($t->doTrackAction('http://vendor.site/'.$this->specialChars.'.html', 'link'));
     }
 
     private function trackVisitTablet(\MatomoTracker $t, $dateTime)
     {
         $t->setForceVisitDateTime($dateTime);
         $t->setUserAgent('Mozilla/5.0 (Linux; U; en-us; KFAPWI Build/JDQ39) AppleWebKit/535.19 (KHTML, like Gecko) Silk/3.8 Safari/535.19 Silk-Accelerated=true');
 
-        $t->setUrlReferrer('http://www.google.com/search?q=product%2042');
-        $t->setUrl('http://example.org/product42');
+        $t->setUrlReferrer('http://www.google.com/search?q=product%2042'.$this->specialChars);
+        $t->setUrl('http://example.org/product42&name='.$this->specialChars);
         $t->setPerformanceTimings(55, 348, 256, 299, 165, 144);
         $t->setDebugStringAppend('bw_bytes=6851');
         $t->setCustomVariable(1, 'custom', 'variable', 'page');
         $t->setCustomDimension('1', '42');
         $t->setCustomDimension('2', '€');
+
         self::checkResponse($t->doTrackPageView('product 42'));
 
         $t->setForceVisitDateTime(Date::factory($dateTime)->addHour(0.09)->getDatetime());
@@ -228,4 +237,13 @@ private function trackDeviceVisit(\MatomoTracker $t, $dateTime, $useragent)
         $t->doTrackContentImpression('product slider', 'product_18.jpg', 'http://example.org/product18');
         $t->doTrackContentImpression('product zoom', 'product_18.jpg', 'http://example.org/product18');
     }
+
+    private function trackCampaignVisit(\MatomoTracker $t, $dateTime)
+    {
+        $t->setForceVisitDateTime($dateTime);
+        $t->setUserAgent('Mozilla/5.0 (iPod; U; CPU iPhone OS 4_2_1 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Mobile/8C148');
+
+        $t->setUrl('http://example.org?pk_campaign='.$this->specialChars);
+        self::checkResponse($t->doTrackPageView('home'));
+    }
 }

plugins/Live/tests/UI/Live_spec.js

@@ -142,16 +142,32 @@ describe("Live", function () {
         expect(await dialog.screenshot()).to.matchImage('visitor_profile_limited');
     });
 
+    it('should show visitor log next page', async function() {
+        await page.goto("?module=CoreHome&action=index&idSite=1&period=year&date=2010-01-03#?idSite=1&period=year&date=2010-01-03&category=General_Visitors&subcategory=Live_VisitorLog");
+
+        await page.waitForNetworkIdle();
+        await page.waitForSelector('.dataTableVizVisitorLog');
+
+        const link = await page.jQuery('.dataTableNext');
+        await link.click();
+        await page.waitForNetworkIdle();
+
+        var report = await page.$('.reporting-page');
+        expect(await report.screenshot()).to.matchImage('visitor_log_page_next');
+    });
+
     it('should show visitor log purge message when purged and no data', async function() {
         testEnvironment.overrideConfig('Deletelogs', 'delete_logs_enable', 1);
         testEnvironment.overrideConfig('Deletelogs', 'delete_logs_older_than', 4000);
         testEnvironment.save();
 
         await page.goto("?module=CoreHome&action=index&idSite=1&period=year&date=2005-01-03#?idSite=1&period=year&date=2005-01-03&category=General_Visitors&subcategory=Live_VisitorLog");
-
         await page.waitForNetworkIdle();
 
         var report = await page.$('.reporting-page');
         expect(await report.screenshot()).to.matchImage('visitor_log_purged');
     });
+
+
+
 });

plugins/Referrers/VisitorDetails.php

@@ -96,7 +96,7 @@ protected function getKeywordPosition()
 
     protected function getReferrerName(): string
     {
-        return urldecode($this->details['referer_name'] ?? '');
+         return html_entity_decode(($this->details['referer_name'] ?? ''), ENT_QUOTES, "UTF-8");
     }
 
     protected function getSearchEngineUrl()