diff --git a/CHANGELOG.md b/CHANGELOG.md index 3075734c5b80..d2a047733274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,10 @@ All notable changes to this project will be documented in this file. - CSV export now includes pageviews, bounce rate and visit duration in addition to visitors plausible/analytics#952 - Send stats to multiple dashboards by configuring a comma-separated list of domains plausible/analytics#968 - Time on Page metric available in detailed Top Pages report plausible/analytics#1007 -- Added `CLICKHOUSE_FLUSH_INTERVAL_MS` and `CLICKHOUSE_MAX_BUFFER_SIZE` configuration parameters +- Glob (wildcard) based page, entry page and exit page filters plausible/analytics#1067 +- Exclusion filters for page, entry page and exit page filters plausible/analytics#1067 +- Menu to add new and edit existing filters directly plausible/analytics#1067 +- Added `CLICKHOUSE_FLUSH_INTERVAL_MS` and `CLICKHOUSE_MAX_BUFFER_SIZE` configuration parameters plausible/analytics#1073 ### Fixed - Fix weekly report time range plausible/analytics#951 diff --git a/assets/css/app.css b/assets/css/app.css index 58376b8b2360..255a904fc74f 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -310,3 +310,11 @@ iframe[hidden] { .pagination-link[disabled] { @apply cursor-default bg-gray-100 dark:bg-gray-300 pointer-events-none; } + +.filter-list-text:hover ~ .filter-list-edit { + display: flex; +} + +.filter-list-text:hover ~ .filter-list-remove { + display: none; +} diff --git a/assets/css/modal.css b/assets/css/modal.css index 10028c9eadc0..2a36314ef6cb 100644 --- a/assets/css/modal.css +++ b/assets/css/modal.css @@ -38,7 +38,6 @@ .modal__container { background-color: #fff; padding: 1rem 2rem; - max-width: 860px; border-radius: 4px; margin: 50px auto; box-sizing: border-box; diff --git a/assets/js/dashboard/datepicker.js b/assets/js/dashboard/datepicker.js index 6fa47e770df0..d3aee1791772 100644 --- a/assets/js/dashboard/datepicker.js +++ b/assets/js/dashboard/datepicker.js @@ -161,10 +161,10 @@ class DatePicker extends React.Component { const leftClasses = `flex items-center px-2 border-r border-gray-300 rounded-l dark:border-gray-500 dark:text-gray-100 ${ - disabledLeft ? "bg-gray-200 dark:bg-gray-900" : "" + disabledLeft ? "bg-gray-300 dark:bg-gray-950" : "hover:bg-gray-200 dark:hover:bg-gray-900" }`; const rightClasses = `flex items-center px-2 rounded-r dark:text-gray-100 ${ - disabledRight ? "bg-gray-200 dark:bg-gray-900" : "" + disabledRight ? "bg-gray-300 dark:bg-gray-950" : "hover:bg-gray-200 dark:hover:bg-gray-900" }`; return (
@@ -242,7 +242,7 @@ class DatePicker extends React.Component { onKeyPress={this.open} className="flex items-center justify-between rounded bg-white dark:bg-gray-800 shadow px-4 pr-3 py-2 leading-tight cursor-pointer text-sm font-medium text-gray-800 - dark:text-gray-200 h-full" + dark:text-gray-200 h-full hover:bg-gray-200 dark:hover:bg-gray-900" tabIndex="0" role="button" aria-haspopup="true" @@ -312,7 +312,7 @@ class DatePicker extends React.Component { to={{from: false, to: false, period, ...opts}} onClick={this.close.bind(this)} query={this.props.query} - className={`${boldClass } px-4 py-2 md:text-sm leading-tight hover:bg-gray-100 + className={`${boldClass } px-4 py-2 md:text-sm leading-tight hover:bg-gray-200 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100 flex items-center justify-between`} > {text} diff --git a/assets/js/dashboard/filters.js b/assets/js/dashboard/filters.js index 48e4e082a392..1fd179918231 100644 --- a/assets/js/dashboard/filters.js +++ b/assets/js/dashboard/filters.js @@ -1,6 +1,6 @@ import React from 'react'; -import { withRouter } from 'react-router-dom' -import { countFilters, navigateToQuery, removeQueryParam } from './query' +import { withRouter, Link } from 'react-router-dom' +import { countFilters, formattedFilters, navigateToQuery, removeQueryParam } from './query' import Datamap from 'datamaps' import Transition from "../transition.js"; @@ -60,7 +60,7 @@ class Filters extends React.Component { } handleKeyup(e) { - const {query, history} = this.props + const { query, history } = this.props if (e.ctrlKey || e.metaKey || e.altKey) return @@ -70,7 +70,7 @@ class Filters extends React.Component { } handleResize() { - this.setState({ viewport: window.innerWidth || 639}); + this.setState({ viewport: window.innerWidth || 639 }); } handleClick(e) { @@ -102,6 +102,9 @@ class Filters extends React.Component { }; filterText(key, value, query) { + const negated = value[0] == '!' && ['page', 'entry_page', 'exit_page'].includes(key) + value = negated ? value.slice(1) : value + if (key === "goal") { return Completed goal {value} } @@ -111,50 +114,50 @@ class Filters extends React.Component { return {eventName}.{metaKey} is {metaValue} } if (key === "source") { - return Source: {value} + return Source is {value} } if (key === "utm_medium") { - return UTM medium: {value} + return UTM medium is {value} } if (key === "utm_source") { - return UTM source: {value} + return UTM source is {value} } if (key === "utm_campaign") { - return UTM campaign: {value} + return UTM campaign is {value} } if (key === "referrer") { - return Referrer: {value} + return Referrer is {value} } if (key === "screen") { - return Screen size: {value} + return Screen size is {value} } if (key === "browser") { - return Browser: {value} + return Browser is {value} } if (key === "browser_version") { const browserName = query.filters["browser"] ? query.filters["browser"] : 'Browser' - return {browserName}.Version: {value} + return {browserName} Version is {value} } if (key === "os") { - return Operating System: {value} + return Operating System is {value} } if (key === "os_version") { const osName = query.filters["os"] ? query.filters["os"] : 'OS' - return {osName}.Version: {value} + return {osName} Version is {value} } if (key === "country") { const allCountries = Datamap.prototype.worldTopo.objects.world.geometries; - const selectedCountry = allCountries.find((c) => c.id === value) || {properties: {name: value}}; - return Country: {selectedCountry.properties.name} + const selectedCountry = allCountries.find((c) => c.id === value) || { properties: { name: value } }; + return Country is {selectedCountry.properties.name} } if (key === "page") { - return Page: {value} + return Page is{negated ? ' not' : ''} {value} } if (key === "entry_page") { - return Entry Page: {value} + return Entry Page is{negated ? ' not' : ''} {value} } if (key === "exit_page") { - return Exit Page: {value} + return Exit Page is{negated ? ' not' : ''} {value} } } @@ -171,18 +174,50 @@ class Filters extends React.Component { } renderDropdownFilter(history, [key, value], query) { + if (['goal', 'props'].includes(key)) { + return ( +
+ {this.filterText(key, value, query)} + this.removeFilter(key, history, query)}>✕ +
+ ) + } + return ( -
- {this.filterText(key, value, query)} - this.removeFilter(key, history, query)}>✕ +
+ + {this.filterText(key, value, query)} + + + this.removeFilter(key, history, query)}>✕
) } renderListFilter(history, [key, value], query) { return ( - - {this.filterText(key, value, query)} this.removeFilter(key, history, query)}>✕ + + {['goal', 'props'].includes(key) ? ( + + {this.filterText(key, value, query)} + + ) : ( + <> + + {this.filterText(key, value, query)} + + + + + + )} + this.removeFilter(key, history, query)}> + + ) } @@ -198,11 +233,15 @@ class Filters extends React.Component { renderDropDownContent() { const { viewport } = this.state; - const { history, query } = this.props; + const { history, query, site } = this.props; return (
this.dropDownNode = node}>
+ + + Add Filter + {this.appliedFilters.map((filter) => this.renderDropdownFilter(history, filter, query))}
this.clearAllFilters(history, query)}> Clear All Filters @@ -239,11 +278,16 @@ class Filters extends React.Component { } renderFilterList() { - const { history, query } = this.props; + const { history, query, site } = this.props; + const { viewport } = this.state; return ( -
+
{(this.appliedFilters.map((filter) => this.renderListFilter(history, filter, query)))} + + + {viewport <= 768 ? "Filter" : "Add Filter"} +
); } @@ -251,15 +295,11 @@ class Filters extends React.Component { render() { const { wrapped, viewport } = this.state; - if (this.appliedFilters.length > 0) { - if (wrapped === 2 || viewport <= 768) { - return this.renderDropDown(); - } - - return this.renderFilterList(); + if (this.appliedFilters.length > 0 && (wrapped === 2 || viewport <= 768)) { + return this.renderDropDown(); } - return null; + return this.renderFilterList(); } } diff --git a/assets/js/dashboard/historical.js b/assets/js/dashboard/historical.js index f2380ac5441e..3723e9028037 100644 --- a/assets/js/dashboard/historical.js +++ b/assets/js/dashboard/historical.js @@ -34,7 +34,7 @@ class Historical extends React.Component {
- +
diff --git a/assets/js/dashboard/query.js b/assets/js/dashboard/query.js index 1fa3c8fdb13d..2c98a90611ed 100644 --- a/assets/js/dashboard/query.js +++ b/assets/js/dashboard/query.js @@ -159,3 +159,22 @@ export function eventName(query) { } return 'pageviews' } + +export const formattedFilters = { + 'goal': 'Goal', + 'props': 'Props', + 'source': 'Source', + 'utm_medium': 'UTM Medium', + 'utm_source': 'UTM Source', + 'utm_campaign': 'UTM Campaign', + 'referrer': 'Referrer', + 'screen': 'Screen size', + 'browser': 'Browser', + 'browser_version': 'Browser Version', + 'os': 'Operating System', + 'os_version': 'Operating System Version', + 'country': 'Country', + 'page': 'Page', + 'entry_page': 'Entry Page', + 'exit_page': 'Exit Page' +} diff --git a/assets/js/dashboard/realtime.js b/assets/js/dashboard/realtime.js index 1a05c8d66bf9..33ce0dbadf9d 100644 --- a/assets/js/dashboard/realtime.js +++ b/assets/js/dashboard/realtime.js @@ -33,7 +33,7 @@ class Realtime extends React.Component {
- +
diff --git a/assets/js/dashboard/router.js b/assets/js/dashboard/router.js index 23d223ba67f2..6f70bb5df25f 100644 --- a/assets/js/dashboard/router.js +++ b/assets/js/dashboard/router.js @@ -7,6 +7,7 @@ import PagesModal from './stats/modals/pages' import EntryPagesModal from './stats/modals/entry-pages' import ExitPagesModal from './stats/modals/exit-pages' import CountriesModal from './stats/modals/countries' +import FilterModal from './stats/modals/filter' import {BrowserRouter, Switch, Route, useLocation} from "react-router-dom"; @@ -50,6 +51,9 @@ export default function Router({site, loggedIn}) { + + + diff --git a/assets/js/dashboard/stats/modals/filter.js b/assets/js/dashboard/stats/modals/filter.js new file mode 100644 index 000000000000..84e513e24772 --- /dev/null +++ b/assets/js/dashboard/stats/modals/filter.js @@ -0,0 +1,137 @@ +import React from "react"; +import { withRouter, Redirect } from 'react-router-dom' + +import Modal from './modal' +import { parseQuery, formattedFilters, navigateToQuery } from '../../query' + +class FilterModal extends React.Component { + constructor(props) { + super(props) + this.state = { + query: parseQuery(props.location.search, props.site), + selectedFilter: "", + negated: false, + updatedValue: "", + filterSaved: false + } + + this.editableGoals = Object.keys(this.state.query.filters).filter(filter => !['goal', 'props'].includes(filter)) + } + + componentDidMount() { + this.setState({ selectedFilter: this.props.match.params.field }) + } + + componentDidUpdate(prevProps, prevState) { + const { query, selectedFilter } = this.state + + if (prevState.selectedFilter !== selectedFilter) { + const negated = query.filters[selectedFilter] && query.filters[selectedFilter][0] == '!' && this.negationSupported(selectedFilter) + const updatedValue = negated ? query.filters[selectedFilter].slice(1) : (query.filters[selectedFilter] || "") + + this.setState({ updatedValue, negated }) + } + } + + negationSupported(filter) { + return ['page', 'entry_page', 'exit_page'].includes(filter) + } + + renderBody() { + const { selectedFilter, negated, updatedValue, query } = this.state; + + const finalFilterValue = (this.negationSupported(selectedFilter) && negated ? '!' : '') + updatedValue + const finalizedQuery = new URLSearchParams(window.location.search) + const validFilter = this.editableGoals.includes(selectedFilter) && updatedValue + finalizedQuery.set(selectedFilter, finalFilterValue) + + return ( + +

{query.filters[selectedFilter] ? 'Edit' : 'Add'} Filter

+ +
+
+
{ + if (validFilter) { + this.setState({ finalizedQuery }) + } + }}> + + + {this.negationSupported(selectedFilter) && +
+ +
+ } + + {selectedFilter && + { this.setState({ updatedValue: e.target.value }) }} + /> + } + + + + {query.filters[selectedFilter] && + + } + +
+
+
+ ) + } + + render() { + const { finalizedQuery } = this.state + + if (finalizedQuery) { + return + } + + return ( + + { this.renderBody()} + + ) + } +} + +export default withRouter(FilterModal) diff --git a/assets/js/dashboard/stats/modals/modal.js b/assets/js/dashboard/stats/modals/modal.js index 292b2d110871..cd6603baa575 100644 --- a/assets/js/dashboard/stats/modals/modal.js +++ b/assets/js/dashboard/stats/modals/modal.js @@ -47,7 +47,7 @@ class Modal extends React.Component {
-
+
{this.props.children}
diff --git a/assets/tailwind.config.js b/assets/tailwind.config.js index a56552449688..4dfba9ebfac7 100644 --- a/assets/tailwind.config.js +++ b/assets/tailwind.config.js @@ -14,6 +14,7 @@ module.exports = { extend: { colors: { orange: colors.orange, + 'gray-950': 'rgb(13, 18, 30)', 'gray-850': 'rgb(26, 32, 44)', 'gray-825': 'rgb(37, 47, 63)' }, @@ -30,7 +31,7 @@ module.exports = { '9': 9, }, maxWidth: { - '2xs': '16rem', + '2xs': '15rem', '3xs': '12rem', } }, diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index e636af7cd225..4aa6a899d98b 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -257,13 +257,7 @@ defmodule Plausible.Stats.Clickhouse do from(s in referrers, where: s.referrer_source != "") end - referrers = - if query.filters["page"] do - page = query.filters["page"] - from(s in referrers, where: s.entry_page == ^page) - else - referrers - end + referrers = apply_page_as_entry_page(referrers, site, query) referrers = if include_details do @@ -326,13 +320,7 @@ defmodule Plausible.Stats.Clickhouse do end defp apply_page_as_entry_page(db_query, _site, query) do - page = query.filters["page"] - - if is_binary(page) do - from(s in db_query, where: s.entry_page == ^page) - else - db_query - end + include_path_filter_entry(db_query, query.filters["page"]) end def utm_mediums(site, query, limit \\ 9, page \\ 1, show_noref \\ false) do @@ -1156,21 +1144,9 @@ defmodule Plausible.Stats.Clickhouse do sessions_q end - sessions_q = - if query.filters["entry_page"] do - entry_page = query.filters["entry_page"] - from(s in sessions_q, where: s.entry_page == ^entry_page) - else - sessions_q - end + sessions_q = include_path_filter_entry(sessions_q, query.filters["entry_page"]) - sessions_q = - if query.filters["exit_page"] do - exit_page = query.filters["exit_page"] - from(s in sessions_q, where: s.exit_page == ^exit_page) - else - sessions_q - end + sessions_q = include_path_filter_exit(sessions_q, query.filters["exit_page"]) q = from(e in "events", @@ -1194,13 +1170,7 @@ defmodule Plausible.Stats.Clickhouse do q end - q = - if query.filters["page"] do - page = query.filters["page"] - from(e in q, where: e.pathname == ^page) - else - q - end + q = include_path_filter(q, query.filters["page"]) if query.filters["props"] do [{key, val}] = query.filters["props"] |> Enum.into([]) @@ -1350,21 +1320,9 @@ defmodule Plausible.Stats.Clickhouse do q end - q = - if query.filters["entry_page"] do - entry_page = query.filters["entry_page"] - from(s in q, where: s.entry_page == ^entry_page) - else - q - end + q = include_path_filter_entry(q, query.filters["entry_page"]) - q = - if query.filters["exit_page"] do - exit_page = query.filters["exit_page"] - from(s in q, where: s.exit_page == ^exit_page) - else - q - end + q = include_path_filter_exit(q, query.filters["exit_page"]) if query.filters["referrer"] do ref = query.filters["referrer"] @@ -1464,13 +1422,7 @@ defmodule Plausible.Stats.Clickhouse do q end - q = - if query.filters["page"] do - page = query.filters["page"] - from(e in q, where: e.pathname == ^page) - else - q - end + q = include_path_filter(q, query.filters["page"]) if query.filters["props"] do [{key, val}] = query.filters["props"] |> Enum.into([]) @@ -1550,12 +1502,9 @@ defmodule Plausible.Stats.Clickhouse do end if path do - if String.match?(path, ~r/\*/) do - path_regex = - "^#{path}\/?$" - |> String.replace(~r/\*\*/, ".*") - |> String.replace(~r/(? String.replace(~r/\*\*/, ".*") + |> String.replace(~r/(? 1, + "bounce_rate" => nil, + "count" => 2, + "pageviews" => 2, + "name" => "/register" + }, + %{ + "time_on_page" => nil, + "bounce_rate" => nil, + "count" => 1, + "pageviews" => 1, + "name" => "/irrelevant" + } + ] + end + + test "filters pages based on exclusion", %{conn: conn, site: site} do + filters = Jason.encode!(%{page: "!/"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01&detailed=true&filters=#{ + filters + }" + ) + + assert json_response(conn, 200) == [ + %{ + "time_on_page" => 1, + "bounce_rate" => nil, + "count" => 2, + "pageviews" => 2, + "name" => "/register" + }, + %{ + "time_on_page" => nil, + "bounce_rate" => nil, + "count" => 1, + "pageviews" => 1, + "name" => "/contact" + }, + %{ + "time_on_page" => nil, + "bounce_rate" => nil, + "count" => 1, + "pageviews" => 1, + "name" => "/irrelevant" + } + ] + end + + test "filters pages based on wildard exclusion", %{conn: conn, site: site} do + filters = Jason.encode!(%{page: "!/*re*"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01&detailed=true&filters=#{ + filters + }" + ) + + assert json_response(conn, 200) == [ + %{ + "time_on_page" => 82800, + "bounce_rate" => 33.0, + "count" => 3, + "pageviews" => 3, + "name" => "/" + }, + %{ + "time_on_page" => nil, + "bounce_rate" => nil, + "count" => 1, + "pageviews" => 1, + "name" => "/contact" + } + ] + end + test "returns top pages in realtime report", %{conn: conn, site: site} do conn = get(conn, "/api/stats/#{site.domain}/pages?period=realtime") @@ -97,6 +191,18 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do } ] end + + test "filters based on exclusion for entry pages", %{conn: conn, site: site} do + filters = Jason.encode!(%{entry_page: "!/"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/entry-pages?period=day&date=2019-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [] + end end describe "GET /api/stats/:domain/exit-pages" do @@ -109,5 +215,17 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do %{"count" => 3, "exits" => 3, "name" => "/", "exit_rate" => 100.0} ] end + + test "filters based on exclusion for entry pages", %{conn: conn, site: site} do + filters = Jason.encode!(%{exit_page: "!/"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/exit-pages?period=day&date=2019-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [] + end end end