From 35f975accb45152dd669e0009237eed3dc6ada2b Mon Sep 17 00:00:00 2001 From: mattab Date: Sat, 20 Apr 2013 21:02:35 +1200 Subject: [PATCH] Fixes #3904: * new segment 'siteSearchKeyword' Fixes #3903, #3905: * adding few fields in the Live API output to accomodate getSuggestedValuesForSegment * renamed other fields for consistency with segment names Fixes #3906: * new API: getSuggestedValuesForSegment which returns top suggested values for a particular segment. It uses the Live.getLastVisitsDetails API to fetch the most recently used values, and will show the most used values first * Adding tests for everything. The test case actually generates data for all segments so that VisitsSummary.get returns some data for each of the 47 segments being tested returns some data. How it works: * generate extended data in fixture * Tests (1) call getSuggestedValuesForSegment for each segment, check there is some data returned for each segment * get the first suggested value from the list, * Tests (2) call VisitsSummary.get with this segment value, eg. countryCode==ru. * I worked this way for all 47 segments until all tests had some data ==> now we know that all segments have been tested and that the auto suggest works for all segments. TDD FTW! --- core/API/DocumentationGenerator.php | 2 + core/API/Request.php | 2 +- core/DataTable.php | 20 + core/DataTable/Filter/ColumnDelete.php | 22 +- core/Segment.php | 17 +- lang/en.php | 1 + libs/PiwikTracker/PiwikTracker.php | 2 +- plugins/API/API.php | 66 +- plugins/Actions/Actions.php | 54 +- plugins/CoreUpdater/CoreUpdater.php | 3 +- plugins/Live/API.php | 130 +- plugins/Live/Visitor.php | 38 +- plugins/Transitions/API.php | 12 +- plugins/UserCountry/UserCountry.php | 13 +- plugins/UserSettings/UserSettings.php | 4 +- .../Core/PluginsFunctions/WidgetsListTest.php | 6 +- tests/PHPUnit/Core/SegmentTest.php | 6 +- tests/PHPUnit/DatabaseTestCase.php | 11 +- .../PHPUnit/Fixtures/ManyVisitsWithGeoIP.php | 45 +- .../Integration/AutoSuggestAPITest.php | 132 ++ .../ManyVisitorsOneWebsiteTest.php | 8 +- ...everalDaysDateRange_ArchivingTestsTest.php | 2 +- .../TrackingAPI_SetVisitorIdTest.php | 1 - ...thCustomVariables_SegmentMatchNONETest.php | 5 + ...PITest__Live.getLastVisitsDetails_year.xml | 1246 +++++++++++++++++ ...ions__API.getSuggestedValuesForSegment.xml | 6 + ...ITest_actions__VisitsSummary.get_range.xml | 12 + ...Code__API.getSuggestedValuesForSegment.xml | 4 + ...t_browserCode__VisitsSummary.get_range.xml | 12 + ...sion__API.getSuggestedValuesForSegment.xml | 4 + ...rowserVersion__VisitsSummary.get_range.xml | 12 + ...city__API.getSuggestedValuesForSegment.xml | 11 + ...tAPITest_city__VisitsSummary.get_range.xml | 12 + ...Code__API.getSuggestedValuesForSegment.xml | 7 + ...continentCode__VisitsSummary.get_range.xml | 12 + ...Code__API.getSuggestedValuesForSegment.xml | 13 + ...t_countryCode__VisitsSummary.get_range.xml | 12 + ...ame1__API.getSuggestedValuesForSegment.xml | 4 + ...VariableName1__VisitsSummary.get_range.xml | 12 + ...ame2__API.getSuggestedValuesForSegment.xml | 2 + ...ame3__API.getSuggestedValuesForSegment.xml | 2 + ...ame4__API.getSuggestedValuesForSegment.xml | 2 + ...ame5__API.getSuggestedValuesForSegment.xml | 4 + ...VariableName5__VisitsSummary.get_range.xml | 12 + ...ame1__API.getSuggestedValuesForSegment.xml | 2 + ...ame2__API.getSuggestedValuesForSegment.xml | 4 + ...ablePageName2__VisitsSummary.get_range.xml | 12 + ...ame3__API.getSuggestedValuesForSegment.xml | 2 + ...ame4__API.getSuggestedValuesForSegment.xml | 4 + ...ablePageName4__VisitsSummary.get_range.xml | 12 + ...ame5__API.getSuggestedValuesForSegment.xml | 4 + ...ablePageName5__VisitsSummary.get_range.xml | 12 + ...lue1__API.getSuggestedValuesForSegment.xml | 2 + ...lue2__API.getSuggestedValuesForSegment.xml | 12 + ...blePageValue2__VisitsSummary.get_range.xml | 12 + ...lue3__API.getSuggestedValuesForSegment.xml | 2 + ...lue4__API.getSuggestedValuesForSegment.xml | 4 + ...blePageValue4__VisitsSummary.get_range.xml | 12 + ...lue5__API.getSuggestedValuesForSegment.xml | 12 + ...blePageValue5__VisitsSummary.get_range.xml | 12 + ...lue1__API.getSuggestedValuesForSegment.xml | 12 + ...ariableValue1__VisitsSummary.get_range.xml | 12 + ...lue2__API.getSuggestedValuesForSegment.xml | 2 + ...lue3__API.getSuggestedValuesForSegment.xml | 2 + ...lue4__API.getSuggestedValuesForSegment.xml | 2 + ...lue5__API.getSuggestedValuesForSegment.xml | 12 + ...ariableValue5__VisitsSummary.get_range.xml | 12 + ...isit__API.getSuggestedValuesForSegment.xml | 6 + ...nceFirstVisit__VisitsSummary.get_range.xml | 12 + ...rder__API.getSuggestedValuesForSegment.xml | 6 + ...commerceOrder__VisitsSummary.get_range.xml | 12 + ...isit__API.getSuggestedValuesForSegment.xml | 6 + ...inceLastVisit__VisitsSummary.get_range.xml | 12 + ...itle__API.getSuggestedValuesForSegment.xml | 6 + ...ntryPageTitle__VisitsSummary.get_range.xml | 12 + ...eUrl__API.getSuggestedValuesForSegment.xml | 5 + ..._entryPageUrl__VisitsSummary.get_range.xml | 12 + ...itle__API.getSuggestedValuesForSegment.xml | 6 + ...exitPageTitle__VisitsSummary.get_range.xml | 12 + ...eUrl__API.getSuggestedValuesForSegment.xml | 5 + ...t_exitPageUrl__VisitsSummary.get_range.xml | 12 + ...tude__API.getSuggestedValuesForSegment.xml | 9 + ...Test_latitude__VisitsSummary.get_range.xml | 12 + ...tude__API.getSuggestedValuesForSegment.xml | 9 + ...est_longitude__VisitsSummary.get_range.xml | 12 + ...Code__API.getSuggestedValuesForSegment.xml | 4 + ...ingSystemCode__VisitsSummary.get_range.xml | 12 + ...itle__API.getSuggestedValuesForSegment.xml | 6 + ...est_pageTitle__VisitsSummary.get_range.xml | 12 + ...eUrl__API.getSuggestedValuesForSegment.xml | 5 + ...ITest_pageUrl__VisitsSummary.get_range.xml | 12 + ...ider__API.getSuggestedValuesForSegment.xml | 4 + ...Test_provider__VisitsSummary.get_range.xml | 12 + ...word__API.getSuggestedValuesForSegment.xml | 4 + ...ferrerKeyword__VisitsSummary.get_range.xml | 12 + ...Name__API.getSuggestedValuesForSegment.xml | 4 + ..._referrerName__VisitsSummary.get_range.xml | 12 + ...Type__API.getSuggestedValuesForSegment.xml | 5 + ..._referrerType__VisitsSummary.get_range.xml | 12 + ...rUrl__API.getSuggestedValuesForSegment.xml | 4 + ...t_referrerUrl__VisitsSummary.get_range.xml | 12 + ...Code__API.getSuggestedValuesForSegment.xml | 11 + ...st_regionCode__VisitsSummary.get_range.xml | 12 + ...tion__API.getSuggestedValuesForSegment.xml | 4 + ...st_resolution__VisitsSummary.get_range.xml | 12 + ...ches__API.getSuggestedValuesForSegment.xml | 5 + ...Test_searches__VisitsSummary.get_range.xml | 12 + ...word__API.getSuggestedValuesForSegment.xml | 4 + ...SearchKeyword__VisitsSummary.get_range.xml | 12 + ...alId__API.getSuggestedValuesForSegment.xml | 5 + ...nvertedGoalId__VisitsSummary.get_range.xml | 12 + ...rted__API.getSuggestedValuesForSegment.xml | 4 + ...isitConverted__VisitsSummary.get_range.xml | 12 + ...ount__API.getSuggestedValuesForSegment.xml | 5 + ...st_visitCount__VisitsSummary.get_range.xml | 12 + ...tion__API.getSuggestedValuesForSegment.xml | 7 + ...visitDuration__VisitsSummary.get_range.xml | 12 + ...atus__API.getSuggestedValuesForSegment.xml | 4 + ...ommerceStatus__VisitsSummary.get_range.xml | 12 + ...itIp__API.getSuggestedValuesForSegment.xml | 20 + ...ITest_visitIp__VisitsSummary.get_range.xml | 12 + ...Hour__API.getSuggestedValuesForSegment.xml | 4 + ...isitLocalHour__VisitsSummary.get_range.xml | 12 + ...Hour__API.getSuggestedValuesForSegment.xml | 5 + ...sitServerHour__VisitsSummary.get_range.xml | 12 + ...orId__API.getSuggestedValuesForSegment.xml | 13 + ...est_visitorId__VisitsSummary.get_range.xml | 12 + ...Type__API.getSuggestedValuesForSegment.xml | 5 + ...t_visitorType__VisitsSummary.get_range.xml | 12 + ...eTest__Live.getLastVisitsDetails_month.xml | 547 ++++++-- ...WebsiteTest__UserCountry.getCity_month.xml | 289 ++-- ...teTest__UserCountry.getContinent_month.xml | 96 +- ...siteTest__UserCountry.getCountry_month.xml | 219 +-- ...bsiteTest__UserCountry.getRegion_month.xml | 269 ++-- ...egment_city__UserCountry.getCity_month.xml | 17 +- ...t_city__UserCountry.getContinent_month.xml | 17 +- ...ent_city__UserCountry.getCountry_month.xml | 17 +- ...ment_city__UserCountry.getRegion_month.xml | 17 +- ...ontinent__UserCountry.getCountry_month.xml | 109 +- ...nt_lat_long__UserCountry.getCity_month.xml | 23 +- ...t_long__UserCountry.getContinent_month.xml | 23 +- ...lat_long__UserCountry.getCountry_month.xml | 23 +- ..._lat_long__UserCountry.getRegion_month.xml | 23 +- ...ment_region__UserCountry.getCity_month.xml | 34 +- ...region__UserCountry.getContinent_month.xml | 17 +- ...t_region__UserCountry.getCountry_month.xml | 17 +- ...nt_region__UserCountry.getRegion_month.xml | 17 +- ...Support__Live.getLastVisitsDetails_day.xml | 16 +- ...ecified__Live.getLastVisitsDetails_day.xml | 13 + ...eportMetadata__API.getSegmentsMetadata.xml | 20 +- ...Ordered__Live.getLastVisitsDetails_day.xml | 56 +- ...thItems__Live.getLastVisitsDetails_day.xml | 187 +-- ...ndNormalAPI__Actions.getPageUrls_range.xml | 4 +- ...stomVariables.getCustomVariables_range.xml | 48 +- ..._MetadataAndNormalAPI__Goals.get_range.xml | 2 +- ...alAPI__Live.getLastVisitsDetails_range.xml | 59 +- ...NormalAPI__Referers.getCampaigns_range.xml | 4 +- ...dNormalAPI__Referers.getKeywords_range.xml | 12 +- ...aAndNormalAPI__VisitsSummary.get_range.xml | 2 +- tests/PHPUnit/IntegrationTestCase.php | 49 +- tests/PHPUnit/Plugins/PrivacyManagerTest.php | 4 +- 161 files changed, 4102 insertions(+), 887 deletions(-) create mode 100644 tests/PHPUnit/Integration/AutoSuggestAPITest.php create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest__Live.getLastVisitsDetails_year.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_actions__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_actions__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_browserCode__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_browserCode__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_browserVersion__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_browserVersion__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_city__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_city__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_continentCode__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_continentCode__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_countryCode__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_countryCode__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableName1__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableName1__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableName2__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableName3__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableName4__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableName5__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableName5__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageName1__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageName2__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageName2__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageName3__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageName4__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageName4__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageName5__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageName5__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageValue1__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageValue2__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageValue2__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageValue3__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageValue4__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageValue4__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageValue5__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariablePageValue5__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableValue1__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableValue1__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableValue2__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableValue3__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableValue4__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableValue5__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_customVariableValue5__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_daysSinceFirstVisit__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_daysSinceFirstVisit__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_daysSinceLastEcommerceOrder__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_daysSinceLastEcommerceOrder__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_daysSinceLastVisit__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_daysSinceLastVisit__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_entryPageTitle__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_entryPageTitle__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_entryPageUrl__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_entryPageUrl__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_exitPageTitle__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_exitPageTitle__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_exitPageUrl__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_exitPageUrl__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_latitude__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_latitude__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_longitude__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_longitude__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_operatingSystemCode__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_operatingSystemCode__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_pageTitle__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_pageTitle__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_pageUrl__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_pageUrl__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_provider__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_provider__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_referrerKeyword__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_referrerKeyword__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_referrerName__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_referrerName__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_referrerType__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_referrerType__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_referrerUrl__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_referrerUrl__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_regionCode__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_regionCode__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_resolution__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_resolution__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_searches__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_searches__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_siteSearchKeyword__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_siteSearchKeyword__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitConvertedGoalId__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitConvertedGoalId__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitConverted__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitConverted__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitCount__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitCount__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitDuration__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitDuration__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitEcommerceStatus__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitEcommerceStatus__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitIp__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitIp__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitLocalHour__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitLocalHour__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitServerHour__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitServerHour__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitorId__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitorId__VisitsSummary.get_range.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitorType__API.getSuggestedValuesForSegment.xml create mode 100644 tests/PHPUnit/Integration/expected/test_AutoSuggestAPITest_visitorType__VisitsSummary.get_range.xml diff --git a/core/API/DocumentationGenerator.php b/core/API/DocumentationGenerator.php index 1c2d9a62a43..ff830133e77 100644 --- a/core/API/DocumentationGenerator.php +++ b/core/API/DocumentationGenerator.php @@ -132,6 +132,7 @@ public function getExampleUrl($class, $methodName, $parametersToSet = array()) 'lastMinutes' => '30', 'abandonedCarts' => '0', 'ip' => '194.57.91.215', +// 'segmentName' => 'browserCode', ); foreach ($parametersToSet as $name => $value) { @@ -231,4 +232,5 @@ public function getParametersString($class, $name) $sParameters = implode(", ", $asParameters); return "($sParameters)"; } + } diff --git a/core/API/Request.php b/core/API/Request.php index 1ce66abf379..b30fd663c16 100644 --- a/core/API/Request.php +++ b/core/API/Request.php @@ -111,7 +111,7 @@ private function sanitizeRequest() * It then calls the API Proxy which will call the requested method. * * @throws Piwik_FrontController_PluginDeactivatedException - * @return mixed The data resulting from the API call + * @return Piwik_DataTable|mixed The data resulting from the API call */ public function process() { diff --git a/core/DataTable.php b/core/DataTable.php index 1784802799a..b5e2064980e 100644 --- a/core/DataTable.php +++ b/core/DataTable.php @@ -675,6 +675,26 @@ public function getColumn($name) return $columnValues; } + /** + * Returns the array containing all rows values of all columns which name starts with $name + * + * @param $name + * @return array + */ + public function getColumnsStartingWith($name) + { + $columnValues = array(); + foreach ($this->getRows() as $row) { + $columns = $row->getColumns(); + foreach($columns as $column => $value) { + if(strpos($column, $name) === 0) { + $columnValues[] = $row->getColumn($column); + } + } + } + return $columnValues; + } + /** * Returns an array containing the rows Metadata values * diff --git a/core/DataTable/Filter/ColumnDelete.php b/core/DataTable/Filter/ColumnDelete.php index 3ac3f5b244a..709d56a02e1 100644 --- a/core/DataTable/Filter/ColumnDelete.php +++ b/core/DataTable/Filter/ColumnDelete.php @@ -33,6 +33,14 @@ class Piwik_DataTable_Filter_ColumnDelete extends Piwik_DataTable_Filter */ private $columnsToKeep; + /** + * Hack: when specifying "showColumns", sometimes we'd like to also keep columns that "look" like a given column, + * without manually specifying all these columns (which may not be possible if column names are generated dynamically) + * + * Column will be kept, if they match any name in the $columnsToKeep, or if they look like anyColumnToKeep__anythingHere + */ + const APPEND_TO_COLUMN_NAME_TO_KEEP = '__'; + /** * Delete the column, only if the value was zero * @@ -101,8 +109,18 @@ public function filter($table) if (!empty($this->columnsToKeep)) { foreach ($table->getRows() as $row) { foreach ($row->getColumns() as $name => $value) { - // label cannot be removed via whitelisting - if ($name != 'label' && !isset($this->columnsToKeep[$name])) { + + $keep = false; + // @see self::APPEND_TO_COLUMN_NAME_TO_KEEP + foreach($this->columnsToKeep as $nameKeep => $true) { + if(strpos($name, $nameKeep . self::APPEND_TO_COLUMN_NAME_TO_KEEP) === 0) { + $keep = true; + } + } + + if (!$keep + && $name != 'label' // label cannot be removed via whitelisting + && !isset($this->columnsToKeep[$name])) { $row->deleteColumn($name); } } diff --git a/core/Segment.php b/core/Segment.php index 7802ba440d3..6ba00c64781 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -58,11 +58,6 @@ public function __construct($string, $idSites) $segment->setSubExpressionsAfterCleanup($cleanedExpressions); } - public function getPrettyString() - { - //@TODO segment.getPrettyString - } - public function isEmpty() { return empty($this->string); @@ -71,16 +66,6 @@ public function isEmpty() protected $availableSegments = array(); protected $segmentsHumanReadable = ''; - private function getUniqueSqlFields() - { - $expressions = $this->segment->parsedSubExpressions; - $uniqueFields = array(); - foreach ($expressions as $expression) { - $uniqueFields[] = $expression[Piwik_SegmentExpression::INDEX_OPERAND][0]; - } - return $uniqueFields; - } - protected function getCleanedExpression($expression) { if (empty($this->availableSegments)) { @@ -114,7 +99,7 @@ protected function getCleanedExpression($expression) if (isset($segment['sqlFilter']) && !empty($segment['sqlFilter']) ) { - $value = call_user_func($segment['sqlFilter'], $value, $segment['sqlSegment'], $matchType); + $value = call_user_func($segment['sqlFilter'], $value, $segment['sqlSegment'], $matchType, $name); // sqlFilter-callbacks might return arrays for more complex cases // e.g. see Piwik_Actions::getIdActionFromSegment() diff --git a/lang/en.php b/lang/en.php index bdba2286c3e..4063febf0b5 100644 --- a/lang/en.php +++ b/lang/en.php @@ -413,6 +413,7 @@ 'Actions_ColumnSearchCategory' => 'Search Category', 'Actions_ColumnSearchResultsCount' => 'Search Results Count', 'Actions_ColumnSearchKeyword' => 'Keyword', + 'Actions_SiteSearchKeyword' => 'Keyword (Site Search)', 'Actions_ColumnClickedURL' => 'Clicked URL', 'Actions_ColumnDownloadURL' => 'Download URL', 'Actions_ColumnEntryPageURL' => 'Entry Page URL', diff --git a/libs/PiwikTracker/PiwikTracker.php b/libs/PiwikTracker/PiwikTracker.php index 97e5acbc82a..7f609317ca1 100644 --- a/libs/PiwikTracker/PiwikTracker.php +++ b/libs/PiwikTracker/PiwikTracker.php @@ -839,7 +839,7 @@ public function setBrowserHasCookies($bool) */ public function setDebugStringAppend($string) { - $this->DEBUG_APPEND_URL = $string; + $this->DEBUG_APPEND_URL = '&' . $string; } /** diff --git a/plugins/API/API.php b/plugins/API/API.php index 5ca4e2156c1..9c69be2d5c7 100644 --- a/plugins/API/API.php +++ b/plugins/API/API.php @@ -78,7 +78,8 @@ public function getCssFiles($notification) *
  • the list of metrics that will be returned by each method, along with their human readable name, via "getDefaultMetrics" and "getDefaultProcessedMetrics"
    • *
  • the list of segments metadata supported by all functions that have a 'segment' parameter
    • *
  • the (truly magic) method "getProcessedReport" will return a human readable version of any other report, and include the processed metrics such as - * conversion rate, time on site, etc. which are not directly available in other methods. + * conversion rate, time on site, etc. which are not directly available in other methods.
    • + *
  • the method "getSuggestedValuesForSegment" returns top suggested values for a particular segment. It uses the Live.getLastVisitsDetails API to fetch the most recently used values, and will return the most often used values first.
    • * * The Metadata API is for example used by the Piwik Mobile App to automatically display all Piwik reports, with translated report & columns names and nicely formatted values. * More information on the Metadata API documentation page @@ -912,7 +913,7 @@ private function hideShowMetrics($columns, $emptyColumns = array()) foreach ($columns as $name => $ignore) { // if the current column should not be kept, remove it $idx = array_search($name, $columnsToKeep); - if ($idx === FALSE) // if $name is not in $columnsToKeep + if ($idx === false) // if $name is not in $columnsToKeep { unset($columns[$name]); } @@ -1625,4 +1626,65 @@ public function getBulkRequest($urls) } return $result; } + + /** + * Given a segment, will return a list of the most used values for this particular segment. + * @param $segmentName + * @param $idSite + * @throws Exception + */ + public function getSuggestedValuesForSegment($segmentName, $idSite) + { + Piwik::checkUserHasViewAccess($idSite); + $maxSuggestionsToReturn = 30; + $segmentsMetadata = $this->getSegmentsMetadata($idSite, $_hideImplementationData = false); + + $segmentFound = false; + foreach($segmentsMetadata as $segmentMetadata) { + if($segmentMetadata['segment'] == $segmentName) { + $segmentFound = $segmentMetadata; + break; + } + } + if(empty($segmentFound)) { + throw new Exception("Requested segment not found."); + } + + $startDate = Piwik_Date::now()->subDay(60)->toString(); + + // we know which SQL field this segment matches to: call the LIVE api to get last 1000 visitors values + $request = new Piwik_API_Request("method=Live.getLastVisitsDetails + &idSite=$idSite + &period=range + &date=$startDate,today + &filter_limit=10000 + &format=original + &serialize=0 + &flat=1 + &segment="); + $table = $request->process(); + if(empty($table)) { + throw new Exception("There was no data to suggest for $segmentName"); + } + + // Cleanup data to return the top suggested (non empty) labels for this segment + $values = $table->getColumn($segmentName); + + // Select also flattened keys (custom variables "page" scope, page URLs for one visit, page titles for one visit) + $valuesBis = $table->getColumnsStartingWith($segmentName . Piwik_DataTable_Filter_ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP); + $values = array_merge($values, $valuesBis); + + // remove false values (while keeping zeros) + $values = array_filter( $values, 'strlen' ); + + // we have a list of all values. let's show the most frequently used first. + $values = array_count_values( $values ); + arsort($values); + $values = array_keys($values); + + $values = array_map( array('Piwik_Common', 'unsanitizeInputValue'), $values); + + $values = array_slice($values, 0, $maxSuggestionsToReturn); + return $values; + } } diff --git a/plugins/Actions/Actions.php b/plugins/Actions/Actions.php index 097d2e3dee0..8a9be015d73 100644 --- a/plugins/Actions/Actions.php +++ b/plugins/Actions/Actions.php @@ -103,7 +103,14 @@ public function getSegmentsMetadata($notification) 'sqlSegment' => 'log_link_visit_action.idaction_name', 'sqlFilter' => $sqlFilter, ); - // TODO here could add keyword segment and hack $sqlFilter to make it select the right idaction + $segments[] = array( + 'type' => 'dimension', + 'category' => 'Actions_Actions', + 'name' => 'Actions_SiteSearchKeyword', + 'segment' => 'siteSearchKeyword', + 'sqlSegment' => 'log_link_visit_action.idaction_name', + 'sqlFilter' => $sqlFilter, + ); } /** @@ -113,30 +120,29 @@ public function getSegmentsMetadata($notification) * Usually, these callbacks only return a value that should be compared to the * column in the database. In this case, that doesn't work since multiple IDs * can match an expression (e.g. "pageUrl=@foo"). - * @param string $string + * @param string $valueToMatch * @param string $sqlField * @param string $matchType * @throws Exception * @return array|int|string */ - public function getIdActionFromSegment($string, $sqlField, $matchType = '==') + public function getIdActionFromSegment($valueToMatch, $sqlField, $matchType, $segmentName) { - // Field is visit_*_idaction_url or visit_*_idaction_name - $actionType = strpos($sqlField, '_name') === false - ? Piwik_Tracker_Action::TYPE_ACTION_URL - : Piwik_Tracker_Action::TYPE_ACTION_NAME; + $actionType = $this->guessActionTypeFromSegment($segmentName); if ($actionType == Piwik_Tracker_Action::TYPE_ACTION_URL) { // for urls trim protocol and www because it is not recorded in the db - $string = preg_replace('@^http[s]?://(www\.)?@i', '', $string); + $valueToMatch = preg_replace('@^http[s]?://(www\.)?@i', '', $valueToMatch); } + $valueToMatch = Piwik_Common::sanitizeInputValue(Piwik_Common::unsanitizeInputValue($valueToMatch)); + // exact matches work by returning the id directly if ($matchType == Piwik_SegmentExpression::MATCH_EQUAL || $matchType == Piwik_SegmentExpression::MATCH_NOT_EQUAL ) { $sql = Piwik_Tracker_Action::getSqlSelectActionId(); - $bind = array($string, $string, $actionType); + $bind = array($valueToMatch, $valueToMatch, $actionType); $idAction = Piwik_FetchOne($sql, $bind); // if the action is not found, we hack -100 to ensure it tries to match against an integer // otherwise binding idaction_name to "false" returns some rows for some reasons (in case &segment=pageTitle==Větrnásssssss) @@ -150,23 +156,24 @@ public function getIdActionFromSegment($string, $sqlField, $matchType = '==') // build the expression based on the match type $sql = 'SELECT idaction FROM ' . Piwik_Common::prefixTable('log_action') . ' WHERE '; + $sqlMatchType = 'AND type = ' . $actionType; switch ($matchType) { case '=@': // use concat to make sure, no %s occurs because some plugins use %s in their sql - $sql .= '( name LIKE CONCAT(\'%\', ?, \'%\') AND type = ' . $actionType . ' )'; + $sql .= '( name LIKE CONCAT(\'%\', ?, \'%\') ' . $sqlMatchType . ' )'; break; case '!@': - $sql .= '( name NOT LIKE CONCAT(\'%\', ?, \'%\') AND type = ' . $actionType . ' )'; + $sql .= '( name NOT LIKE CONCAT(\'%\', ?, \'%\') ' . $sqlMatchType . ' )'; break; default: - throw new Exception("This match type is not available for action-segments."); + throw new Exception("This match type $matchType is not available for action-segments."); break; } return array( // mark that the returned value is an sql-expression instead of a literal value 'SQL' => $sql, - 'bind' => $string + 'bind' => $valueToMatch, ); } @@ -600,5 +607,26 @@ static protected function isCustomVariablesPluginsEnabled() { return Piwik_PluginsManager::getInstance()->isPluginActivated('CustomVariables'); } + + /** + * @param $segmentName + * @return int + * @throws Exception + */ + protected function guessActionTypeFromSegment($segmentName) + { + if (stripos($segmentName, 'pageurl') !== false) { + $actionType = Piwik_Tracker_Action::TYPE_ACTION_URL; + return $actionType; + } elseif (stripos($segmentName, 'pagetitle') !== false) { + $actionType = Piwik_Tracker_Action::TYPE_ACTION_NAME; + return $actionType; + } elseif (stripos($segmentName, 'sitesearch') !== false) { + $actionType = Piwik_Tracker_Action::TYPE_SITE_SEARCH; + return $actionType; + } else { + throw new Exception(" The segment $segmentName has an unexpected value."); + } + } } diff --git a/plugins/CoreUpdater/CoreUpdater.php b/plugins/CoreUpdater/CoreUpdater.php index 30a28f76a62..865c3da4adb 100644 --- a/plugins/CoreUpdater/CoreUpdater.php +++ b/plugins/CoreUpdater/CoreUpdater.php @@ -69,7 +69,8 @@ function dispatch() && $action == 'saveLanguage') ) { if (Piwik_FrontController::shouldRethrowException()) { - throw new Exception("Piwik and/or some plugins have been upgraded to a new version. Please run the update process first. See documentation: http://piwik.org/docs/update/"); + throw new Exception("Piwik and/or some plugins have been upgraded to a new version. \n". + "--> Please run the update process first. See documentation: http://piwik.org/docs/update/ \n"); } else { Piwik::redirectToModule('CoreUpdater'); } diff --git a/plugins/Live/API.php b/plugins/Live/API.php index 919afb0682a..5cb3043839f 100644 --- a/plugins/Live/API.php +++ b/plugins/Live/API.php @@ -97,13 +97,15 @@ public function getCounters($idSite, $lastMinutes, $segment = false) * @param int $visitorId * @param int $idSite * @param int $filter_limit + * @param bool $flat Whether to flatten the visitor details array + * * @return Piwik_DataTable */ - public function getLastVisitsForVisitor($visitorId, $idSite, $filter_limit = 10) + public function getLastVisitsForVisitor($visitorId, $idSite, $filter_limit = 10, $flat = false) { Piwik::checkUserHasViewAccess($idSite); $visitorDetails = $this->loadLastVisitorDetailsFromDatabase($idSite, $period = false, $date = false, $segment = false, $filter_limit, $filter_offset = false, $visitorId); - $table = $this->getCleanedVisitorsFromDetails($visitorDetails, $idSite); + $table = $this->getCleanedVisitorsFromDetails($visitorDetails, $idSite, $flat); return $table; } @@ -121,25 +123,23 @@ public function getLastVisitsForVisitor($visitorId, $idSite, $filter_limit = 10) * * @return Piwik_DataTable */ - public function getLastVisitsDetails($idSite, $period, $date, $segment = false, $filter_limit = false, $filter_offset = false, $minTimestamp = false) + public function getLastVisitsDetails($idSite, $period, $date, $segment = false, $filter_limit = false, $filter_offset = false, $minTimestamp = false, $flat = false) { if (empty($filter_limit)) { $filter_limit = 10; } Piwik::checkUserHasViewAccess($idSite); $visitorDetails = $this->loadLastVisitorDetailsFromDatabase($idSite, $period, $date, $segment, $filter_limit, $filter_offset, $visitorId = false, $minTimestamp); - $dataTable = $this->getCleanedVisitorsFromDetails($visitorDetails, $idSite); - + $dataTable = $this->getCleanedVisitorsFromDetails($visitorDetails, $idSite, $flat); return $dataTable; } /** * @deprecated */ - public function getLastVisits($idSite, $filter_limit = 10, $minTimestamp = false) { - return $this->getLastVisitsDetails($idSite, $period = false, $date = false, $segment = false, $filter_limit, $filter_offset = false, $minTimestamp); + return $this->getLastVisitsDetails($idSite, $period = false, $date = false, $segment = false, $filter_limit, $filter_offset = false, $minTimestamp, $flat = false); } /** @@ -147,9 +147,10 @@ public function getLastVisits($idSite, $filter_limit = 10, $minTimestamp = false * as well as make the data human readable * @param array $visitorDetails * @param int $idSite + * @param bool $flat whether to flatten the array (eg. 'customVariables' names/values will appear in the root array rather than in 'customVariables' key * @return Piwik_DataTable */ - private function getCleanedVisitorsFromDetails($visitorDetails, $idSite) + private function getCleanedVisitorsFromDetails($visitorDetails, $idSite, $flat = false) { $actionsLimit = (int)Piwik_Config::getInstance()->General['visitor_log_maximum_actions_per_visit']; @@ -200,6 +201,7 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite) LEFT JOIN " . Piwik_Common::prefixTable('log_action') . " AS log_action_title ON log_link_visit_action.idaction_name = log_action_title.idaction WHERE log_link_visit_action.idvisit = ? + ORDER BY server_time ASC LIMIT 0, $actionsLimit "; $actionDetails = Piwik_FetchAll($sql, array($idvisit)); @@ -212,8 +214,8 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite) $cvarKey = $actionDetail['custom_var_k' . $i]; $cvarKey = $this->getCustomVariablePrettyKey($cvarKey); $customVariablesPage[$i] = array( - 'customVariableName' . $i => $cvarKey, - 'customVariableValue' . $i => $actionDetail['custom_var_v' . $i], + 'customVariablePageName' . $i => $cvarKey, + 'customVariablePageValue' . $i => $actionDetail['custom_var_v' . $i], ); } unset($actionDetail['custom_var_k' . $i]); @@ -222,21 +224,30 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite) if (!empty($customVariablesPage)) { $actionDetail['customVariables'] = $customVariablesPage; } - // reconstruct url from prefix + + // Reconstruct url from prefix $actionDetail['url'] = Piwik_Tracker_Action::reconstructNormalizedUrl($actionDetail['url'], $actionDetail['url_prefix']); unset($actionDetail['url_prefix']); - // set the time spent for this action (which is the timeSpentRef of the next action) + + // Set the time spent for this action (which is the timeSpentRef of the next action) if (isset($actionDetails[$actionIdx + 1])) { $actionDetail['timeSpent'] = $actionDetails[$actionIdx + 1]['timeSpentRef']; $actionDetail['timeSpentPretty'] = Piwik::getPrettyTimeFromSeconds($actionDetail['timeSpent']); } unset($actionDetails[$actionIdx]['timeSpentRef']); // not needed after timeSpent is added - // handle generation time + + // Handle generation time if ($actionDetail['custom_float'] > 0) { $actionDetail['generationTime'] = Piwik::getPrettyTimeFromSeconds($actionDetail['custom_float'] / 1000); } unset($actionDetail['custom_float']); + + // Handle Site Search + if($actionDetail['type'] == Piwik_Tracker_Action::TYPE_SITE_SEARCH) { + $actionDetail['siteSearchKeyword'] = $actionDetail['pageTitle']; + unset($actionDetail['pageTitle']); + } } // If the visitor converted a goal, we shall select all Goals @@ -244,6 +255,7 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite) SELECT 'goal' as type, goal.name as goalName, + goal.idgoal as goalId, goal.revenue as revenue, log_conversion.idlink_va as goalPageId, log_conversion.server_time as serverTimePretty, @@ -256,6 +268,7 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite) AND goal.deleted = 0 WHERE log_conversion.idvisit = ? AND log_conversion.idgoal > 0 + ORDER BY server_time ASC LIMIT 0, $actionsLimit "; $goalDetails = Piwik_FetchAll($sql, array($idvisit)); @@ -274,6 +287,7 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite) FROM " . Piwik_Common::prefixTable('log_conversion') . " AS log_conversion WHERE idvisit = ? AND idgoal <= " . Piwik_Tracker_GoalManager::IDGOAL_ORDER . " + ORDER BY server_time ASC LIMIT 0, $actionsLimit"; $ecommerceDetails = Piwik_FetchAll($sql, array($idvisit)); @@ -365,9 +379,13 @@ private function getCleanedVisitorsFromDetails($visitorDetails, $idSite) // Convert datetimes to the site timezone $dateTimeVisit = Piwik_Date::factory($details['serverTimePretty'], $timezone); $details['serverTimePretty'] = $dateTimeVisit->getLocalized(Piwik_Translate('CoreHome_ShortDateFormat') . ' %time%'); + } $visitorDetailsArray['goalConversions'] = count($goalDetails); + if($flat) { + $visitorDetailsArray = $this->flattenVisitorDetailsArray($visitorDetailsArray); + } $table->addRowFromArray(array(Piwik_DataTable_Row::COLUMNS => $visitorDetailsArray)); } return $table; @@ -385,6 +403,92 @@ private function getCustomVariablePrettyKey($key) return $key; } + /** + * The &flat=1 feature is used by API.getSuggestedValuesForSegment + * + * @param $visitorDetailsArray + * @return array + */ + private function flattenVisitorDetailsArray($visitorDetailsArray) + { + // flatten visit custom variables + if (is_array($visitorDetailsArray['customVariables'])) { + foreach ($visitorDetailsArray['customVariables'] as $thisCustomVar) { + $visitorDetailsArray = array_merge($visitorDetailsArray, $thisCustomVar); + } + unset($visitorDetailsArray['customVariables']); + } + + // flatten page views custom variables + $count = 1; + foreach ($visitorDetailsArray['actionDetails'] as $action) { + if (!empty($action['customVariables'])) { + foreach ($action['customVariables'] as $thisCustomVar) { + foreach ($thisCustomVar as $cvKey => $cvValue) { + $flattenedKeyName = $cvKey . Piwik_DataTable_Filter_ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $cvValue; + $count++; + } + } + } + } + // Flatten Goals + $count = 1; + foreach($visitorDetailsArray['actionDetails'] as $action) { + if(!empty($action['goalId'])) { + $flattenedKeyName = 'visitConvertedGoalId' . Piwik_DataTable_Filter_ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $action['goalId']; + $count++; + } + } + + // Flatten Page Titles/URLs + $count = 1; + foreach($visitorDetailsArray['actionDetails'] as $action) { + if(!empty($action['url'])) { + $flattenedKeyName = 'pageUrl' . Piwik_DataTable_Filter_ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $action['url']; + } + + if(!empty($action['pageTitle'])) { + $flattenedKeyName = 'pageTitle' . Piwik_DataTable_Filter_ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $action['pageTitle']; + } + + if(!empty($action['siteSearchKeyword'])) { + $flattenedKeyName = 'siteSearchKeyword' . Piwik_DataTable_Filter_ColumnDelete::APPEND_TO_COLUMN_NAME_TO_KEEP . $count; + $visitorDetailsArray[$flattenedKeyName] = $action['siteSearchKeyword']; + } + $count++; + } + + // Entry/exit pages + $firstAction = $lastAction = false; + foreach($visitorDetailsArray['actionDetails'] as $action) { + if($action['type'] == 'action') { + if(empty($firstAction)) { + $firstAction = $action; + } + $lastAction = $action; + } + } + + if(!empty($firstAction['pageTitle'])) { + $visitorDetailsArray['entryPageTitle'] = $firstAction['pageTitle']; + } + if(!empty($firstAction['url'])) { + $visitorDetailsArray['entryPageUrl'] = $firstAction['url']; + } + if(!empty($lastAction['pageTitle'])) { + $visitorDetailsArray['exitPageTitle'] = $lastAction['pageTitle']; + } + if(!empty($lastAction['url'])) { + $visitorDetailsArray['exitPageUrl'] = $lastAction['url']; + } + + return $visitorDetailsArray; + } + private function sortByServerTime($a, $b) { $ta = strtotime($a['serverTimePretty']); diff --git a/plugins/Live/Visitor.php b/plugins/Live/Visitor.php index aaa4e8a61da..45a3ec7f080 100644 --- a/plugins/Live/Visitor.php +++ b/plugins/Live/Visitor.php @@ -60,6 +60,8 @@ function getAllVisitorDetails() // all time entries 'serverDate' => $this->getServerDate(), 'visitLocalTime' => $this->getVisitLocalTime(), + 'visitLocalHour' => $this->getVisitLocalHour(), + 'visitServerHour' => $this->getVisitServerHour(), 'firstActionTimestamp' => $this->getTimestampFirstAction(), 'lastActionTimestamp' => $this->getTimestampLastAction(), 'lastActionDateTime' => $this->getDateTimeLastAction(), @@ -77,6 +79,7 @@ function getAllVisitorDetails() 'countryCode' => $this->getCountryCode(), 'countryFlag' => $this->getCountryFlag(), 'region' => $this->getRegionName(), + 'regionCode' => $this->getRegionCode(), 'city' => $this->getCityName(), 'location' => $this->getPrettyLocation(), 'latitude' => $this->getLatitude(), @@ -93,12 +96,15 @@ function getAllVisitorDetails() 'referrerSearchEngineUrl' => $this->getSearchEngineUrl(), 'referrerSearchEngineIcon' => $this->getSearchEngineIcon(), 'operatingSystem' => $this->getOperatingSystem(), + 'operatingSystemCode' => $this->getOperatingSystemCode(), 'operatingSystemShortName' => $this->getOperatingSystemShortName(), 'operatingSystemIcon' => $this->getOperatingSystemIcon(), 'browserFamily' => $this->getBrowserFamily(), 'browserFamilyDescription' => $this->getBrowserFamilyDescription(), 'browserName' => $this->getBrowser(), 'browserIcon' => $this->getBrowserIcon(), + 'browserCode' => $this->getBrowserCode(), + 'browserVersion' => $this->getBrowserVersion(), 'screenType' => $this->getScreenType(), 'resolution' => $this->getResolution(), 'screenTypeIcon' => $this->getScreenTypeIcon(), @@ -120,6 +126,16 @@ function getVisitLocalTime() return $this->details['visitor_localtime']; } + function getVisitServerHour() + { + return date('G', strtotime($this->details['visit_last_action_time'])); + } + + function getVisitLocalHour() + { + return date('G', strtotime('2012-12-21 ' . $this->details['visitor_localtime'])); + } + function getVisitCount() { return $this->details['visitor_count_visits']; @@ -249,7 +265,7 @@ function getCityName() public function getRegionName() { - $region = $this->details['location_region']; + $region = $this->getRegionCode(); if ($region != '' && $region != Piwik_Tracker_Visit::UNKNOWN_CODE) { return Piwik_UserCountry_LocationProvider_GeoIp::getRegionNameFromCodes( $this->details['location_country'], $region); @@ -257,6 +273,11 @@ public function getRegionName() return null; } + public function getRegionCode() + { + return $this->details['location_region']; + } + function getPrettyLocation() { $parts = array(); @@ -430,6 +451,11 @@ function getPluginIcons() return null; } + function getOperatingSystemCode() + { + return $this->details['config_os']; + } + function getOperatingSystem() { return Piwik_getOSLabel($this->details['config_os']); @@ -455,6 +481,16 @@ function getBrowserFamily() return Piwik_getBrowserFamily($this->details['config_browser_name']); } + function getBrowserCode() + { + return $this->details['config_browser_name']; + } + + function getBrowserVersion() + { + return $this->details['config_browser_version']; + } + function getBrowser() { return Piwik_getBrowserLabel($this->details['config_browser_name'] . ";" . $this->details['config_browser_version']); diff --git a/plugins/Transitions/API.php b/plugins/Transitions/API.php index 80afa16d945..4457d71046e 100644 --- a/plugins/Transitions/API.php +++ b/plugins/Transitions/API.php @@ -134,25 +134,25 @@ private function deriveIdAction($actionName, $actionType) case 'url': $originalActionName = $actionName; $actionName = Piwik_Common::unsanitizeInputValue($actionName); - $id = $actionsPlugin->getIdActionFromSegment($actionName, 'idaction_url'); + $id = $actionsPlugin->getIdActionFromSegment($actionName, 'idaction_url', Piwik_SegmentExpression::MATCH_EQUAL, 'pageUrl'); if ($id < 0) { // an example where this is needed is urls containing < or > $actionName = $originalActionName; - $id = $actionsPlugin->getIdActionFromSegment($actionName, 'idaction_url'); + $id = $actionsPlugin->getIdActionFromSegment($actionName, 'idaction_url', Piwik_SegmentExpression::MATCH_EQUALs, 'pageUrl'); } return $id; case 'title': - $id = $actionsPlugin->getIdActionFromSegment($actionName, 'idaction_name'); + $id = $actionsPlugin->getIdActionFromSegment($actionName, 'idaction_name', Piwik_SegmentExpression::MATCH_EQUAL, 'pageTitle'); if ($id < 0) { - $unkown = Piwik_Actions_ArchivingHelper::getUnknownActionName( + $unknown = Piwik_Actions_ArchivingHelper::getUnknownActionName( Piwik_Tracker_Action::TYPE_ACTION_NAME); - if (trim($actionName) == trim($unkown)) { - $id = $actionsPlugin->getIdActionFromSegment('', 'idaction_name'); + if (trim($actionName) == trim($unknown)) { + $id = $actionsPlugin->getIdActionFromSegment('', 'idaction_name', Piwik_SegmentExpression::MATCH_EQUAL, 'pageTitle'); } } diff --git a/plugins/UserCountry/UserCountry.php b/plugins/UserCountry/UserCountry.php index 20d2935248a..34003af178b 100644 --- a/plugins/UserCountry/UserCountry.php +++ b/plugins/UserCountry/UserCountry.php @@ -164,7 +164,7 @@ public function getSegmentsMetadata($notification) 'type' => 'dimension', 'category' => 'Visit Location', 'name' => Piwik_Translate('UserCountry_Country'), - 'segment' => 'country', + 'segment' => 'countryCode', 'sqlSegment' => 'log_visit.location_country', 'acceptedValues' => 'de, us, fr, in, es, etc.', ); @@ -172,7 +172,7 @@ public function getSegmentsMetadata($notification) 'type' => 'dimension', 'category' => 'Visit Location', 'name' => Piwik_Translate('UserCountry_Continent'), - 'segment' => 'continent', + 'segment' => 'continentCode', 'sqlSegment' => 'log_visit.location_country', 'acceptedValues' => 'eur, asi, amc, amn, ams, afr, ant, oce', 'sqlFilter' => array('Piwik_UserCountry', 'getCountriesForContinent'), @@ -181,7 +181,7 @@ public function getSegmentsMetadata($notification) 'type' => 'dimension', 'category' => 'Visit Location', 'name' => Piwik_Translate('UserCountry_Region'), - 'segment' => 'region', + 'segment' => 'regionCode', 'sqlSegment' => 'log_visit.location_region', 'acceptedValues' => '01 02, OR, P8, etc.
    eg. region=A1;country=fr', ); @@ -197,7 +197,7 @@ public function getSegmentsMetadata($notification) 'type' => 'dimension', 'category' => 'Visit Location', 'name' => Piwik_Translate('UserCountry_Latitude'), - 'segment' => 'lat', + 'segment' => 'latitude', 'sqlSegment' => 'log_visit.location_latitude', 'acceptedValues' => '-33.578, 40.830, etc.
    You can select visitors within a lat/long range using &segment=lat>X;lat<Y;long>M;long<N.', ); @@ -205,7 +205,7 @@ public function getSegmentsMetadata($notification) 'type' => 'dimension', 'category' => 'Visit Location', 'name' => Piwik_Translate('UserCountry_Longitude'), - 'segment' => 'long', + 'segment' => 'longitude', 'sqlSegment' => 'log_visit.location_longitude', 'acceptedValues' => '-70.664, 14.326, etc.', ); @@ -489,9 +489,8 @@ private function setLongCityRegionId(&$row) */ public static function getCountriesForContinent($continent) { - $continent = strtolower($continent); - $result = array(); + $continent = strtolower($continent); foreach (Piwik_Common::getCountriesList() as $countryCode => $continentCode) { if ($continent == $continentCode) { $result[] = $countryCode; diff --git a/plugins/UserSettings/UserSettings.php b/plugins/UserSettings/UserSettings.php index d46b952c567..90be882806a 100644 --- a/plugins/UserSettings/UserSettings.php +++ b/plugins/UserSettings/UserSettings.php @@ -59,7 +59,7 @@ public function getInformation() 'UserSettings', 'getBrowser', 'UserSettings_ColumnBrowser', - 'browserName', + 'browserCode', 'log_visit.config_browser_name', 'FF, IE, CH, SF, OP, etc.', null,), @@ -110,7 +110,7 @@ public function getInformation() 'UserSettings', 'getOS', 'UserSettings_ColumnOperatingSystem', - 'operatingSystem', + 'operatingSystemCode', 'log_visit.config_os', 'WXP, WI7, MAC, LIN, AND, IPD, etc.', null,), diff --git a/tests/PHPUnit/Core/PluginsFunctions/WidgetsListTest.php b/tests/PHPUnit/Core/PluginsFunctions/WidgetsListTest.php index bd7983cdf6e..7c2c0e598c9 100644 --- a/tests/PHPUnit/Core/PluginsFunctions/WidgetsListTest.php +++ b/tests/PHPUnit/Core/PluginsFunctions/WidgetsListTest.php @@ -24,9 +24,7 @@ public function testGet() $_GET['idSite'] = 1; - $pluginsManager = Piwik_PluginsManager::getInstance(); - $pluginsToLoad = Piwik_Config::getInstance()->Plugins['Plugins']; - $pluginsManager->loadPlugins($pluginsToLoad); + IntegrationTestCase::loadAllPlugins(); Piwik_WidgetsList::_reset(); $widgets = Piwik_GetWidgetsList(); @@ -52,6 +50,7 @@ public function testGet() foreach ($numberOfWidgets AS $category => $widgetCount) { $this->assertEquals($widgetCount, count($widgets[$category]), sprintf("Widget: %s", $category)); } + IntegrationTestCase::unloadAllPlugins(); } /** @@ -132,4 +131,5 @@ public function testGetWithGoalsAndEcommerce() } } + } diff --git a/tests/PHPUnit/Core/SegmentTest.php b/tests/PHPUnit/Core/SegmentTest.php index 2d021ea65cc..35da2a58a32 100644 --- a/tests/PHPUnit/Core/SegmentTest.php +++ b/tests/PHPUnit/Core/SegmentTest.php @@ -44,17 +44,17 @@ public function getCommonTestData() { return array( // Normal segment - array('country==France', array( + array('countryCode==France', array( 'where' => ' log_visit.location_country = ? ', 'bind' => array('France'))), // unescape the comma please - array('country==a\,==', array( + array('countryCode==a\,==', array( 'where' => ' log_visit.location_country = ? ', 'bind' => array('a,=='))), // AND, with 2 values rewrites - array('country==a;visitorType!=returning;visitorType==new', array( + array('countryCode==a;visitorType!=returning;visitorType==new', array( 'where' => ' log_visit.location_country = ? AND log_visit.visitor_returning <> ? AND log_visit.visitor_returning = ? ', 'bind' => array('a', '1', '0'))), diff --git a/tests/PHPUnit/DatabaseTestCase.php b/tests/PHPUnit/DatabaseTestCase.php index 2b0627f251b..9f91c7910c5 100644 --- a/tests/PHPUnit/DatabaseTestCase.php +++ b/tests/PHPUnit/DatabaseTestCase.php @@ -15,6 +15,7 @@ */ class DatabaseTestCase extends PHPUnit_Framework_TestCase { + /** * Setup the database and create the base tables for all tests */ @@ -58,14 +59,7 @@ public function setUp() public function tearDown() { parent::tearDown(); - try { - $plugins = Piwik_PluginsManager::getInstance()->getLoadedPlugins(); - foreach ($plugins AS $plugin) { - $plugin->uninstall(); - } - Piwik_PluginsManager::getInstance()->unloadPlugins(); - } catch (Exception $e) { - } + IntegrationTestCase::unloadAllPlugins(); Piwik::dropDatabase(); Piwik_DataTable_Manager::getInstance()->deleteAll(); Piwik_Option::getInstance()->clearCache(); @@ -76,4 +70,5 @@ public function tearDown() Piwik_TablePartitioning::$tablesAlreadyInstalled = null; Zend_Registry::_unsetInstance(); } + } diff --git a/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php b/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php index 33f2f7006cc..88cd783ae12 100644 --- a/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php +++ b/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php @@ -30,6 +30,9 @@ class Test_Piwik_Fixture_ManyVisitsWithGeoIP extends Test_Piwik_BaseFixture '103.29.196.229', // in Indonesia (Bali), (only Indonesia will show up) ); + protected $idGoal; + protected $idGoal2; + public function setUp() { $this->setUpWebsitesAndGoals(); @@ -58,7 +61,8 @@ public function tearDown() private function setUpWebsitesAndGoals() { self::createWebsite($this->dateTime, 0, "Site 1"); - Piwik_Goals_API::getInstance()->addGoal($this->idSite, 'all', 'url', 'http', 'contains', false, 5); + $this->idGoal = Piwik_Goals_API::getInstance()->addGoal($this->idSite, 'all', 'url', 'http', 'contains', false, 5); + $this->idGoal2 = Piwik_Goals_API::getInstance()->addGoal($this->idSite, 'two', 'url', 'xxxxxxxxxxxxx', 'contains', false, 5); } private function trackVisits($visitorCount, $setIp = false, $useLocal = true, $doBulk = false) @@ -73,7 +77,7 @@ private function trackVisits($visitorCount, $setIp = false, $useLocal = true, $d $t->setTokenAuth(self::getTokenAuth()); } for ($i = 0; $i != $visitorCount; ++$i) { - $t->setNewVisitorId(); + $t->setVisitorId( substr(md5($i + 1000), 0, $t::LENGTH_VISITOR_ID)); if ($setIp) { $t->setIp(current($this->ips)); next($this->ips); @@ -85,7 +89,12 @@ private function trackVisits($visitorCount, $setIp = false, $useLocal = true, $d $date = Piwik_Date::factory($dateTime)->addDay($i); $t->setForceVisitDateTime($date->getDatetime()); $t->setUrl("http://piwik.net/grue/lair"); - $r = $t->doTrackPageView('It\'s pitch black...'); + $t->setCustomVariable(1, 'Cvar 1 name', 'Cvar1 value is ' .$i , 'visit'); + $t->setCustomVariable(5, 'Cvar 5 name', 'Cvar5 value is ' .$i , 'visit'); + $t->setCustomVariable(2, 'Cvar 2 PAGE name', 'Cvar2 PAGE value is ' .$i, 'page'); + $t->setCustomVariable(5, 'Cvar 5 PAGE name', 'Cvar5 PAGE value is ' .$i, 'page'); + + $r = $t->doTrackPageView('It\'s