Fixes #3904:

 * new segment 'siteSearchKeyword'
Fixes #3903, #3905:
 * adding few fields in the Live API output to accomodate getSuggestedValuesForSegment
 * renamed other fields for consistency with segment names
Fixes #3906:
 * new API: getSuggestedValuesForSegment which returns top suggested values for a particular segment. It uses the Live.getLastVisitsDetails API to fetch the most recently used values, and will show the most used values first

 * Adding tests for everything. The test case actually generates data for all segments so that VisitsSummary.get returns some data for each of the 47 segments being tested returns some data.
  How it works:
  * generate extended data in fixture
  * Tests (1) call getSuggestedValuesForSegment for each segment, check there is some data returned for each segment
  * get the first suggested value from the list,
  * Tests (2) call VisitsSummary.get with this segment value, eg. countryCode==ru.
    * I worked this way for all 47 segments until all tests had some data ==> now we know that all segments have been tested and that the auto suggest works for all segments. TDD FTW!

core/API/DocumentationGenerator.php

@@ -132,6 +132,7 @@ public function getExampleUrl($class, $methodName, $parametersToSet = array())
             'lastMinutes'    => '30',
             'abandonedCarts' => '0',
             'ip'             => '194.57.91.215',
+//            'segmentName'    => 'browserCode',
         );
 
         foreach ($parametersToSet as $name => $value) {
@@ -231,4 +232,5 @@ public function getParametersString($class, $name)
         $sParameters = implode(", ", $asParameters);
         return "($sParameters)";
     }
+
 }

core/API/Request.php

@@ -111,7 +111,7 @@ private function sanitizeRequest()
      * It then calls the API Proxy which will call the requested method.
      *
      * @throws Piwik_FrontController_PluginDeactivatedException
-     * @return mixed  The data resulting from the API call
+     * @return Piwik_DataTable|mixed  The data resulting from the API call
      */
     public function process()
     {

core/DataTable.php

@@ -675,6 +675,26 @@ public function getColumn($name)
         return $columnValues;
     }
 
+    /**
+     * Returns  the array containing all rows values of all columns which name starts with $name
+     *
+     * @param $name
+     * @return array
+     */
+    public function getColumnsStartingWith($name)
+    {
+        $columnValues = array();
+        foreach ($this->getRows() as $row) {
+            $columns = $row->getColumns();
+            foreach($columns as $column => $value) {
+                if(strpos($column, $name) === 0) {
+                    $columnValues[] = $row->getColumn($column);
+                }
+            }
+        }
+        return $columnValues;
+    }
+
     /**
      * Returns an array containing the rows Metadata values
      *

core/DataTable/Filter/ColumnDelete.php

@@ -33,6 +33,14 @@ class Piwik_DataTable_Filter_ColumnDelete extends Piwik_DataTable_Filter
      */
     private $columnsToKeep;
 
+    /**
+     * Hack: when specifying "showColumns", sometimes we'd like to also keep columns that "look" like a given column,
+     * without manually specifying all these columns (which may not be possible if column names are generated dynamically)
+     *
+     * Column will be kept, if they match any name in the $columnsToKeep, or if they look like anyColumnToKeep__anythingHere
+     */
+    const APPEND_TO_COLUMN_NAME_TO_KEEP = '__';
+
     /**
      * Delete the column, only if the value was zero
      *
@@ -101,8 +109,18 @@ public function filter($table)
         if (!empty($this->columnsToKeep)) {
             foreach ($table->getRows() as $row) {
                 foreach ($row->getColumns() as $name => $value) {
-                    // label cannot be removed via whitelisting
-                    if ($name != 'label' && !isset($this->columnsToKeep[$name])) {
+
+                    $keep = false;
+                    // @see self::APPEND_TO_COLUMN_NAME_TO_KEEP
+                    foreach($this->columnsToKeep as $nameKeep => $true) {
+                        if(strpos($name, $nameKeep . self::APPEND_TO_COLUMN_NAME_TO_KEEP) === 0) {
+                            $keep = true;
+                        }
+                    }
+
+                    if (!$keep
+                        && $name != 'label' // label cannot be removed via whitelisting
+                        && !isset($this->columnsToKeep[$name])) {
                         $row->deleteColumn($name);
                     }
                 }

core/Segment.php

@@ -58,11 +58,6 @@ public function __construct($string, $idSites)
         $segment->setSubExpressionsAfterCleanup($cleanedExpressions);
     }
 
-    public function getPrettyString()
-    {
-        //@TODO segment.getPrettyString
-    }
-
     public function isEmpty()
     {
         return empty($this->string);
@@ -71,16 +66,6 @@ public function isEmpty()
     protected $availableSegments = array();
     protected $segmentsHumanReadable = '';
 
-    private function getUniqueSqlFields()
-    {
-        $expressions = $this->segment->parsedSubExpressions;
-        $uniqueFields = array();
-        foreach ($expressions as $expression) {
-            $uniqueFields[] = $expression[Piwik_SegmentExpression::INDEX_OPERAND][0];
-        }
-        return $uniqueFields;
-    }
-
     protected function getCleanedExpression($expression)
     {
         if (empty($this->availableSegments)) {
@@ -114,7 +99,7 @@ protected function getCleanedExpression($expression)
             if (isset($segment['sqlFilter'])
                 && !empty($segment['sqlFilter'])
             ) {
-                $value = call_user_func($segment['sqlFilter'], $value, $segment['sqlSegment'], $matchType);
+                $value = call_user_func($segment['sqlFilter'], $value, $segment['sqlSegment'], $matchType, $name);
 
                 // sqlFilter-callbacks might return arrays for more complex cases
                 // e.g. see Piwik_Actions::getIdActionFromSegment()

lang/en.php

@@ -413,6 +413,7 @@
 	'Actions_ColumnSearchCategory' => 'Search Category',
 	'Actions_ColumnSearchResultsCount' => 'Search Results Count',
 	'Actions_ColumnSearchKeyword' => 'Keyword',
+	'Actions_SiteSearchKeyword' => 'Keyword (Site Search)',
 	'Actions_ColumnClickedURL' => 'Clicked URL',
 	'Actions_ColumnDownloadURL' => 'Download URL',
 	'Actions_ColumnEntryPageURL' => 'Entry Page URL',

libs/PiwikTracker/PiwikTracker.php

@@ -839,7 +839,7 @@ public function setBrowserHasCookies($bool)
      */
     public function setDebugStringAppend($string)
     {
-        $this->DEBUG_APPEND_URL = $string;
+        $this->DEBUG_APPEND_URL = '&' . $string;
     }
 
     /**

plugins/API/API.php

@@ -78,7 +78,8 @@ public function getCssFiles($notification)
  * <li>the list of metrics that will be returned by each method, along with their human readable name, via "getDefaultMetrics" and "getDefaultProcessedMetrics"</li>
  * <li>the list of segments metadata supported by all functions that have a 'segment' parameter</li>
  * <li>the (truly magic) method "getProcessedReport" will return a human readable version of any other report, and include the processed metrics such as
- * conversion rate, time on site, etc. which are not directly available in other methods.
+ * conversion rate, time on site, etc. which are not directly available in other methods.</li>
+ * <li>the method "getSuggestedValuesForSegment" returns top suggested values for a particular segment. It uses the Live.getLastVisitsDetails API to fetch the most recently used values, and will return the most often used values first.</li>
  * </ul>
  * The Metadata API is for example used by the Piwik Mobile App to automatically display all Piwik reports, with translated report & columns names and nicely formatted values.
  * More information on the <a href='http://piwik.org/docs/analytics-api/metadata/' target='_blank'>Metadata API documentation page</a>
@@ -912,7 +913,7 @@ private function hideShowMetrics($columns, $emptyColumns = array())
             foreach ($columns as $name => $ignore) {
                 // if the current column should not be kept, remove it
                 $idx = array_search($name, $columnsToKeep);
-                if ($idx === FALSE) // if $name is not in $columnsToKeep
+                if ($idx === false) // if $name is not in $columnsToKeep
                 {
                     unset($columns[$name]);
                 }
@@ -1625,4 +1626,65 @@ public function getBulkRequest($urls)
         }
         return $result;
     }
+
+    /**
+     * Given a segment, will return a list of the most used values for this particular segment.
+     * @param $segmentName
+     * @param $idSite
+     * @throws Exception
+     */
+    public function getSuggestedValuesForSegment($segmentName, $idSite)
+    {
+        Piwik::checkUserHasViewAccess($idSite);
+        $maxSuggestionsToReturn = 30;
+        $segmentsMetadata = $this->getSegmentsMetadata($idSite, $_hideImplementationData = false);
+
+        $segmentFound = false;
+        foreach($segmentsMetadata as $segmentMetadata) {
+            if($segmentMetadata['segment'] == $segmentName) {
+                $segmentFound = $segmentMetadata;
+                break;
+            }
+        }
+        if(empty($segmentFound)) {
+            throw new Exception("Requested segment not found.");
+        }
+
+        $startDate = Piwik_Date::now()->subDay(60)->toString();
+
+        // we know which SQL field this segment matches to: call the LIVE api to  get last 1000 visitors values
+        $request = new Piwik_API_Request("method=Live.getLastVisitsDetails
+            &idSite=$idSite
+            &period=range
+            &date=$startDate,today
+            &filter_limit=10000
+            &format=original
+            &serialize=0
+            &flat=1
+            &segment=");
+        $table = $request->process();
+        if(empty($table)) {
+            throw new Exception("There was no data to suggest for $segmentName");
+        }
+
+        // Cleanup data to return the top suggested (non empty) labels for this segment
+        $values = $table->getColumn($segmentName);
+
+        // Select also flattened keys (custom variables "page" scope, page URLs for one visit, page titles for one visit)
+        $valuesBis = $table->getColumnsStartingWith($segmentName . Piwik_DataTable_Filter_ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP);
+        $values = array_merge($values, $valuesBis);
+
+        // remove false values (while keeping zeros)
+        $values = array_filter( $values, 'strlen' );
+
+        // we have a list of all values. let's show the most frequently used first.
+        $values = array_count_values( $values );
+        arsort($values);
+        $values = array_keys($values);
+
+        $values = array_map( array('Piwik_Common', 'unsanitizeInputValue'), $values);
+
+        $values = array_slice($values, 0, $maxSuggestionsToReturn);
+        return $values;
+    }
 }

plugins/Actions/Actions.php

@@ -103,7 +103,14 @@ public function getSegmentsMetadata($notification)
             'sqlSegment' => 'log_link_visit_action.idaction_name',
             'sqlFilter'  => $sqlFilter,
         );
-        // TODO here could add keyword segment and hack $sqlFilter to make it select the right idaction
+        $segments[] = array(
+            'type'       => 'dimension',
+            'category'   => 'Actions_Actions',
+            'name'       => 'Actions_SiteSearchKeyword',
+            'segment'    => 'siteSearchKeyword',
+            'sqlSegment' => 'log_link_visit_action.idaction_name',
+            'sqlFilter'  => $sqlFilter,
+        );
     }
 
     /**
@@ -113,30 +120,29 @@ public function getSegmentsMetadata($notification)
      * Usually, these callbacks only return a value that should be compared to the
      * column in the database. In this case, that doesn't work since multiple IDs
      * can match an expression (e.g. "pageUrl=@foo").
-     * @param string $string
+     * @param string $valueToMatch
      * @param string $sqlField
      * @param string $matchType
      * @throws Exception
      * @return array|int|string
      */
-    public function getIdActionFromSegment($string, $sqlField, $matchType = '==')
+    public function getIdActionFromSegment($valueToMatch, $sqlField, $matchType, $segmentName)
     {
-        // Field is visit_*_idaction_url or visit_*_idaction_name
-        $actionType = strpos($sqlField, '_name') === false
-            ? Piwik_Tracker_Action::TYPE_ACTION_URL
-            : Piwik_Tracker_Action::TYPE_ACTION_NAME;
+        $actionType = $this->guessActionTypeFromSegment($segmentName);
 
         if ($actionType == Piwik_Tracker_Action::TYPE_ACTION_URL) {
             // for urls trim protocol and www because it is not recorded in the db
-            $string = preg_replace('@^http[s]?://(www\.)?@i', '', $string);
+            $valueToMatch = preg_replace('@^http[s]?://(www\.)?@i', '', $valueToMatch);
         }
 
+        $valueToMatch = Piwik_Common::sanitizeInputValue(Piwik_Common::unsanitizeInputValue($valueToMatch));
+
         // exact matches work by returning the id directly
         if ($matchType == Piwik_SegmentExpression::MATCH_EQUAL
             || $matchType == Piwik_SegmentExpression::MATCH_NOT_EQUAL
         ) {
             $sql = Piwik_Tracker_Action::getSqlSelectActionId();
-            $bind = array($string, $string, $actionType);
+            $bind = array($valueToMatch, $valueToMatch, $actionType);
             $idAction = Piwik_FetchOne($sql, $bind);
             // if the action is not found, we hack -100 to ensure it tries to match against an integer
             // otherwise binding idaction_name to "false" returns some rows for some reasons (in case &segment=pageTitle==Větrnásssssss)
@@ -150,23 +156,24 @@ public function getIdActionFromSegment($string, $sqlField, $matchType = '==')
 
         // build the expression based on the match type
         $sql = 'SELECT idaction FROM ' . Piwik_Common::prefixTable('log_action') . ' WHERE ';
+        $sqlMatchType = 'AND type = ' . $actionType;
         switch ($matchType) {
             case '=@':
                 // use concat to make sure, no %s occurs because some plugins use %s in their sql
-                $sql .= '( name LIKE CONCAT(\'%\', ?, \'%\') AND type = ' . $actionType . ' )';
+                $sql .= '( name LIKE CONCAT(\'%\', ?, \'%\') ' . $sqlMatchType . ' )';
                 break;
             case '!@':
-                $sql .= '( name NOT LIKE CONCAT(\'%\', ?, \'%\') AND type = ' . $actionType . ' )';
+                $sql .= '( name NOT LIKE CONCAT(\'%\', ?, \'%\') ' . $sqlMatchType . ' )';
                 break;
             default:
-                throw new Exception("This match type is not available for action-segments.");
+                throw new Exception("This match type $matchType is not available for action-segments.");
                 break;
         }
 
         return array(
             // mark that the returned value is an sql-expression instead of a literal value
             'SQL'  => $sql,
-            'bind' => $string
+            'bind' => $valueToMatch,
         );
     }
 
@@ -600,5 +607,26 @@ static protected function isCustomVariablesPluginsEnabled()
     {
         return Piwik_PluginsManager::getInstance()->isPluginActivated('CustomVariables');
     }
+
+    /**
+     * @param $segmentName
+     * @return int
+     * @throws Exception
+     */
+    protected function guessActionTypeFromSegment($segmentName)
+    {
+        if (stripos($segmentName, 'pageurl') !== false) {
+            $actionType = Piwik_Tracker_Action::TYPE_ACTION_URL;
+            return $actionType;
+        } elseif (stripos($segmentName, 'pagetitle') !== false) {
+            $actionType = Piwik_Tracker_Action::TYPE_ACTION_NAME;
+            return $actionType;
+        } elseif (stripos($segmentName, 'sitesearch') !== false) {
+            $actionType = Piwik_Tracker_Action::TYPE_SITE_SEARCH;
+            return $actionType;
+        } else {
+            throw new Exception(" The segment $segmentName has an unexpected value.");
+        }
+    }
 }
 

plugins/CoreUpdater/CoreUpdater.php

@@ -69,7 +69,8 @@ function dispatch()
                 && $action == 'saveLanguage')
         ) {
             if (Piwik_FrontController::shouldRethrowException()) {
-                throw new Exception("Piwik and/or some plugins have been upgraded to a new version. Please run the update process first. See documentation: http://piwik.org/docs/update/");
+                throw new Exception("Piwik and/or some plugins have been upgraded to a new version. \n".
+                    "--> Please run the update process first. See documentation: http://piwik.org/docs/update/ \n");
             } else {
                 Piwik::redirectToModule('CoreUpdater');
             }