diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index f79556d04376..07de7266e627 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -59,12 +59,12 @@ defmodule Plausible.Site do change(site, has_stats: has_stats_val) end - def start_import(site, imported_source) do + def start_import(site, imported_source, status \\ "importing") do change(site, imported_data: %Plausible.Site.ImportedData{ end_date: Timex.today(), source: imported_source, - status: "importing" + status: status } ) end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 93c282f936e0..791e040e4f0a 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -205,7 +205,6 @@ defmodule Plausible.Stats.Breakdown do |> merge_imported(site, query, property, metrics) |> apply_pagination(pagination) |> ClickhouseRepo.all() - # TODO: migrate schema field to 'os' |> transform_keys(%{operating_system: :os}) end @@ -221,7 +220,6 @@ defmodule Plausible.Stats.Breakdown do |> merge_imported(site, query, property, metrics) |> apply_pagination(pagination) |> ClickhouseRepo.all() - # TODO: migrate schema field to 'os' |> transform_keys(%{operating_system: :os}) end diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 444dedec6637..d79aca415a28 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -5,23 +5,46 @@ defmodule Plausible.Stats.Imported do @no_ref "Direct / None" - def timeseries(site, query) do - result = + def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{with_imported: false}, _), + do: native_q + + def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{filters: filters}, _) + when length(filters) > 0, + do: native_q + + def merge_imported_timeseries( + native_q, + %Plausible.Site{id: site_id, imported_data: %{status: "ok"}}, + query, + metrics + ) do + imported_q = from(v in "imported_visitors", - group_by: fragment("date"), - where: v.site_id == ^site.id, + where: v.site_id == ^site_id, where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last, - select: %{ - visitors: sum(v.visitors), - date: fragment("? as date", v.date) - } + select: %{visitors: sum(v.visitors)} ) - |> ClickhouseRepo.all() - |> Enum.map(fn row -> {row[:date], row[:visitors]} end) - |> Map.new() + |> apply_interval(query) + + from(s in Ecto.Query.subquery(native_q), + full_join: i in subquery(imported_q), + on: field(s, :date) == field(i, :date) + ) + |> select_joined_metrics(metrics) + end + + def merge_imported_timeseries(native_q, _site, _query, _metrics), do: native_q - Enum.into(query.date_range, []) - |> Enum.map(fn step -> Map.get(result, step, 0) end) + defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}) do + imported_q + |> group_by([i], fragment("toStartOfMonth(?)", i.date)) + |> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)}) + end + + defp apply_interval(imported_q, _query) do + imported_q + |> group_by([i], i.date) + |> select_merge([i], %{date: i.date}) end def merge_imported(q, %Plausible.Site{imported_data: nil}, _, _, _), do: q @@ -185,6 +208,7 @@ defmodule Plausible.Stats.Imported do on: field(s, ^dim) == field(i, ^dim) ) |> select_joined_metrics(metrics) + |> apply_order_by(metrics) case dim do :source -> @@ -354,9 +378,6 @@ defmodule Plausible.Stats.Imported do |> select_merge([s, i], %{ :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) }) - |> order_by([s, i], - desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) - ) |> select_joined_metrics(rest) end @@ -410,4 +431,11 @@ defmodule Plausible.Stats.Imported do q |> select_joined_metrics(rest) end + + defp apply_order_by(q, [:visitors | rest]) do + order_by(q, [s, i], desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors)) + |> apply_order_by(rest) + end + + defp apply_order_by(q, _), do: q end diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 90fbd8a3ebe5..694a7dd21ace 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -28,27 +28,25 @@ defmodule Plausible.Stats.Timeseries do end) end + defp events_timeseries(_, _, []), do: [] + defp events_timeseries(site, query, metrics) do - from(e in base_event_query(site, query), - group_by: fragment("date"), - order_by: fragment("date"), - select: %{} - ) + from(e in base_event_query(site, query), select: %{}) |> select_bucket(site, query) |> select_event_metrics(metrics) + |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() end + defp sessions_timeseries(_, _, []), do: [] + defp sessions_timeseries(site, query, metrics) do query = Query.treat_page_filter_as_entry_page(query) - from(e in query_sessions(site, query), - group_by: fragment("date"), - order_by: fragment("date"), - select: %{} - ) + from(e in query_sessions(site, query), select: %{}) |> select_bucket(site, query) |> select_session_metrics(metrics) + |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() end @@ -81,8 +79,10 @@ defmodule Plausible.Stats.Timeseries do def select_bucket(q, site, %Query{interval: "month"}) do from( e in q, + group_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone), + order_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone), select_merge: %{ - date: fragment("toStartOfMonth(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone) } ) end @@ -90,8 +90,10 @@ defmodule Plausible.Stats.Timeseries do def select_bucket(q, site, %Query{interval: "date"}) do from( e in q, + group_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone), + order_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone), select_merge: %{ - date: fragment("toDate(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone) } ) end @@ -99,8 +101,10 @@ defmodule Plausible.Stats.Timeseries do def select_bucket(q, site, %Query{interval: "hour"}) do from( e in q, + group_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone), + order_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone), select_merge: %{ - date: fragment("toStartOfHour(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone) } ) end @@ -108,8 +112,10 @@ defmodule Plausible.Stats.Timeseries do def select_bucket(q, _site, %Query{interval: "minute"}) do from( e in q, + group_by: fragment("dateDiff('minute', now(), ?)", e.timestamp), + order_by: fragment("dateDiff('minute', now(), ?)", e.timestamp), select_merge: %{ - date: fragment("dateDiff('minute', now(), ?) as date", e.timestamp) + date: fragment("dateDiff('minute', now(), ?)", e.timestamp) } ) end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index cd559fafd88d..f7716c5c09c9 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -23,25 +23,7 @@ defmodule PlausibleWeb.Api.StatsController do plot = Enum.map(timeseries_result, fn row -> row[:visitors] end) labels = Enum.map(timeseries_result, fn row -> row[:date] end) present_index = present_index_for(site, query, labels) - - {plot, with_imported, source} = - if query.with_imported && site.imported_data do - # Showing imported data. - plot = - Stats.Imported.timeseries(site, timeseries_query) - |> Enum.zip_with(plot, &(&1 + &2)) - - {plot, true, site.imported_data.source} - else - if Enum.any?(query.filters) do - # Hiding imported data due to filtering. - # Setting source to "" hides imported indicator from main graph. - {plot, false, ""} - else - # Hiding imported data either by request or because there is none. - {plot, false, (site.imported_data && site.imported_data.source) || ""} - end - end + with_imported = query.with_imported && site.imported_data && Enum.empty?(query.filters) json(conn, %{ plot: plot, @@ -51,7 +33,7 @@ defmodule PlausibleWeb.Api.StatsController do interval: query.interval, sample_percent: sample_percent, with_imported: with_imported, - imported_source: source + imported_source: site.imported_data && site.imported_data.source }) end diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index ceefa0f58777..0d8fa0216e5a 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -107,7 +107,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&filters=#{filters}" ) - assert %{"plot" => plot, "imported_source" => ""} = json_response(conn, 200) + assert %{"plot" => plot} = json_response(conn, 200) assert Enum.count(plot) == 31 assert List.first(plot) == 1 @@ -115,6 +115,28 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert Enum.sum(plot) == 2 end + test "displays visitors for 6 months with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-06-30 00:00:00]), + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-06-30]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 6 + assert List.first(plot) == 2 + assert List.last(plot) == 2 + assert Enum.sum(plot) == 4 + end + # TODO: missing 6, 12 months, 30 days end diff --git a/test/support/factory.ex b/test/support/factory.ex index 2978638cb726..747d888a66ac 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -180,12 +180,10 @@ defmodule Plausible.Factory do } end - @today Timex.today() |> Timex.to_date() - def imported_visitors_factory do %{ table: "imported_visitors", - date: @today, + date: Timex.today(), visitors: 1, pageviews: 1, bounces: 0, @@ -197,7 +195,7 @@ defmodule Plausible.Factory do def imported_sources_factory do %{ table: "imported_sources", - date: @today, + date: Timex.today(), source: "", visitors: 1, visits: 1, @@ -209,7 +207,7 @@ defmodule Plausible.Factory do def imported_pages_factory do %{ table: "imported_pages", - date: @today, + date: Timex.today(), page: "", visitors: 1, pageviews: 1, @@ -220,7 +218,7 @@ defmodule Plausible.Factory do def imported_entry_pages_factory do %{ table: "imported_entry_pages", - date: @today, + date: Timex.today(), entry_page: "", visitors: 1, entrances: 1, @@ -232,7 +230,7 @@ defmodule Plausible.Factory do def imported_exit_pages_factory do %{ table: "imported_exit_pages", - date: @today, + date: Timex.today(), exit_page: "", visitors: 1, exits: 1 @@ -242,7 +240,7 @@ defmodule Plausible.Factory do def imported_locations_factory do %{ table: "imported_locations", - date: @today, + date: Timex.today(), country: "", region: "", city: 0, @@ -256,7 +254,7 @@ defmodule Plausible.Factory do def imported_devices_factory do %{ table: "imported_devices", - date: @today, + date: Timex.today(), device: "", visitors: 1, visits: 1, @@ -268,7 +266,7 @@ defmodule Plausible.Factory do def imported_browsers_factory do %{ table: "imported_browsers", - date: @today, + date: Timex.today(), browser: "", visitors: 1, visits: 1, @@ -280,7 +278,7 @@ defmodule Plausible.Factory do def imported_operating_systems_factory do %{ table: "imported_operating_systems", - date: @today, + date: Timex.today(), operating_system: "", visitors: 1, visits: 1, diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index 2cb6ebfa2123..e0fdeb094664 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -14,7 +14,7 @@ defmodule Plausible.TestUtils do def add_imported_data(%{site: site}) do site = site - |> Plausible.Site.set_imported_source("Google Analytics") + |> Plausible.Site.start_import("Google Analytics", "ok") |> Repo.update!() {:ok, site: site}