Fix month interval with imported data

lib/plausible/site/schema.ex

@@ -59,12 +59,12 @@ defmodule Plausible.Site do
     change(site, has_stats: has_stats_val)
   end
 
-  def start_import(site, imported_source) do
+  def start_import(site, imported_source, status \\ "importing") do
     change(site,
       imported_data: %Plausible.Site.ImportedData{
         end_date: Timex.today(),
         source: imported_source,
-        status: "importing"
+        status: status
       }
     )
   end

lib/plausible/stats/breakdown.ex

@@ -205,7 +205,6 @@ defmodule Plausible.Stats.Breakdown do
     |> merge_imported(site, query, property, metrics)
     |> apply_pagination(pagination)
     |> ClickhouseRepo.all()
-    # TODO: migrate schema field to 'os'
     |> transform_keys(%{operating_system: :os})
   end
 
@@ -221,7 +220,6 @@ defmodule Plausible.Stats.Breakdown do
     |> merge_imported(site, query, property, metrics)
     |> apply_pagination(pagination)
     |> ClickhouseRepo.all()
-    # TODO: migrate schema field to 'os'
     |> transform_keys(%{operating_system: :os})
   end
 

lib/plausible/stats/imported.ex

@@ -5,23 +5,46 @@ defmodule Plausible.Stats.Imported do
 
   @no_ref "Direct / None"
 
-  def timeseries(site, query) do
-    result =
+  def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{with_imported: false}, _),
+    do: native_q
+
+  def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{filters: filters}, _)
+      when length(filters) > 0,
+      do: native_q
+
+  def merge_imported_timeseries(
+        native_q,
+        %Plausible.Site{id: site_id, imported_data: %{status: "ok"}},
+        query,
+        metrics
+      ) do
+    imported_q =
       from(v in "imported_visitors",
-        group_by: fragment("date"),
-        where: v.site_id == ^site.id,
+        where: v.site_id == ^site_id,
         where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last,
-        select: %{
-          visitors: sum(v.visitors),
-          date: fragment("? as date", v.date)
-        }
+        select: %{visitors: sum(v.visitors)}
       )
-      |> ClickhouseRepo.all()
-      |> Enum.map(fn row -> {row[:date], row[:visitors]} end)
-      |> Map.new()
+      |> apply_interval(query)
+
+    from(s in Ecto.Query.subquery(native_q),
+      full_join: i in subquery(imported_q),
+      on: field(s, :date) == field(i, :date)
+    )
+    |> select_joined_metrics(metrics)
+  end
+
+  def merge_imported_timeseries(native_q, _site, _query, _metrics), do: native_q
 
-    Enum.into(query.date_range, [])
-    |> Enum.map(fn step -> Map.get(result, step, 0) end)
+  defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}) do
+    imported_q
+    |> group_by([i], fragment("toStartOfMonth(?)", i.date))
+    |> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)})
+  end
+
+  defp apply_interval(imported_q, _query) do
+    imported_q
+    |> group_by([i], i.date)
+    |> select_merge([i], %{date: i.date})
   end
 
   def merge_imported(q, %Plausible.Site{imported_data: nil}, _, _, _), do: q
@@ -185,6 +208,7 @@ defmodule Plausible.Stats.Imported do
         on: field(s, ^dim) == field(i, ^dim)
       )
       |> select_joined_metrics(metrics)
+      |> apply_order_by(metrics)
 
     case dim do
       :source ->
@@ -354,9 +378,6 @@ defmodule Plausible.Stats.Imported do
     |> select_merge([s, i], %{
       :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors)
     })
-    |> order_by([s, i],
-      desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors)
-    )
     |> select_joined_metrics(rest)
   end
 
@@ -410,4 +431,11 @@ defmodule Plausible.Stats.Imported do
     q
     |> select_joined_metrics(rest)
   end
+
+  defp apply_order_by(q, [:visitors | rest]) do
+    order_by(q, [s, i], desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors))
+    |> apply_order_by(rest)
+  end
+
+  defp apply_order_by(q, _), do: q
 end

lib/plausible/stats/timeseries.ex

@@ -28,27 +28,25 @@ defmodule Plausible.Stats.Timeseries do
     end)
   end
 
+  defp events_timeseries(_, _, []), do: []
+
   defp events_timeseries(site, query, metrics) do
-    from(e in base_event_query(site, query),
-      group_by: fragment("date"),
-      order_by: fragment("date"),
-      select: %{}
-    )
+    from(e in base_event_query(site, query), select: %{})
     |> select_bucket(site, query)
     |> select_event_metrics(metrics)
+    |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
     |> ClickhouseRepo.all()
   end
 
+  defp sessions_timeseries(_, _, []), do: []
+
   defp sessions_timeseries(site, query, metrics) do
     query = Query.treat_page_filter_as_entry_page(query)
 
-    from(e in query_sessions(site, query),
-      group_by: fragment("date"),
-      order_by: fragment("date"),
-      select: %{}
-    )
+    from(e in query_sessions(site, query), select: %{})
     |> select_bucket(site, query)
     |> select_session_metrics(metrics)
+    |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
     |> ClickhouseRepo.all()
   end
 
@@ -81,35 +79,43 @@ defmodule Plausible.Stats.Timeseries do
   def select_bucket(q, site, %Query{interval: "month"}) do
     from(
       e in q,
+      group_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
+      order_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
       select_merge: %{
-        date: fragment("toStartOfMonth(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone)
+        date: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone)
       }
     )
   end
 
   def select_bucket(q, site, %Query{interval: "date"}) do
     from(
       e in q,
+      group_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
+      order_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
       select_merge: %{
-        date: fragment("toDate(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone)
+        date: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone)
       }
     )
   end
 
   def select_bucket(q, site, %Query{interval: "hour"}) do
     from(
       e in q,
+      group_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
+      order_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone),
       select_merge: %{
-        date: fragment("toStartOfHour(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone)
+        date: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone)
       }
     )
   end
 
   def select_bucket(q, _site, %Query{interval: "minute"}) do
     from(
       e in q,
+      group_by: fragment("dateDiff('minute', now(), ?)", e.timestamp),
+      order_by: fragment("dateDiff('minute', now(), ?)", e.timestamp),
       select_merge: %{
-        date: fragment("dateDiff('minute', now(), ?) as date", e.timestamp)
+        date: fragment("dateDiff('minute', now(), ?)", e.timestamp)
       }
     )
   end

lib/plausible_web/controllers/api/stats_controller.ex

@@ -23,25 +23,7 @@ defmodule PlausibleWeb.Api.StatsController do
     plot = Enum.map(timeseries_result, fn row -> row[:visitors] end)
     labels = Enum.map(timeseries_result, fn row -> row[:date] end)
     present_index = present_index_for(site, query, labels)
-
-    {plot, with_imported, source} =
-      if query.with_imported && site.imported_data do
-        # Showing imported data.
-        plot =
-          Stats.Imported.timeseries(site, timeseries_query)
-          |> Enum.zip_with(plot, &(&1 + &2))
-
-        {plot, true, site.imported_data.source}
-      else
-        if Enum.any?(query.filters) do
-          # Hiding imported data due to filtering.
-          # Setting source to "" hides imported indicator from main graph.
-          {plot, false, ""}
-        else
-          # Hiding imported data either by request or because there is none.
-          {plot, false, (site.imported_data && site.imported_data.source) || ""}
-        end
-      end
+    with_imported = query.with_imported && site.imported_data && Enum.empty?(query.filters)
 
     json(conn, %{
       plot: plot,
@@ -51,7 +33,7 @@ defmodule PlausibleWeb.Api.StatsController do
       interval: query.interval,
       sample_percent: sample_percent,
       with_imported: with_imported,
-      imported_source: source
+      imported_source: site.imported_data && site.imported_data.source
     })
   end
 

test/plausible_web/controllers/api/stats_controller/main_graph_test.exs

@@ -107,14 +107,36 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
           "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&filters=#{filters}"
         )
 
-      assert %{"plot" => plot, "imported_source" => ""} = json_response(conn, 200)
+      assert %{"plot" => plot} = json_response(conn, 200)
 
       assert Enum.count(plot) == 31
       assert List.first(plot) == 1
       assert List.last(plot) == 1
       assert Enum.sum(plot) == 2
     end
 
+    test "displays visitors for 6 months with imported data", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+        build(:pageview, timestamp: ~N[2021-06-30 00:00:00]),
+        build(:imported_visitors, date: ~D[2021-01-01]),
+        build(:imported_visitors, date: ~D[2021-06-30])
+      ])
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true"
+        )
+
+      assert %{"plot" => plot} = json_response(conn, 200)
+
+      assert Enum.count(plot) == 6
+      assert List.first(plot) == 2
+      assert List.last(plot) == 2
+      assert Enum.sum(plot) == 4
+    end
+
     # TODO: missing 6, 12 months, 30 days
   end
 

test/support/factory.ex

@@ -180,12 +180,10 @@ defmodule Plausible.Factory do
     }
   end
 
-  @today Timex.today() |> Timex.to_date()
-
   def imported_visitors_factory do
     %{
       table: "imported_visitors",
-      date: @today,
+      date: Timex.today(),
       visitors: 1,
       pageviews: 1,
       bounces: 0,
@@ -197,7 +195,7 @@ defmodule Plausible.Factory do
   def imported_sources_factory do
     %{
       table: "imported_sources",
-      date: @today,
+      date: Timex.today(),
       source: "",
       visitors: 1,
       visits: 1,
@@ -209,7 +207,7 @@ defmodule Plausible.Factory do
   def imported_pages_factory do
     %{
       table: "imported_pages",
-      date: @today,
+      date: Timex.today(),
       page: "",
       visitors: 1,
       pageviews: 1,
@@ -220,7 +218,7 @@ defmodule Plausible.Factory do
   def imported_entry_pages_factory do
     %{
       table: "imported_entry_pages",
-      date: @today,
+      date: Timex.today(),
       entry_page: "",
       visitors: 1,
       entrances: 1,
@@ -232,7 +230,7 @@ defmodule Plausible.Factory do
   def imported_exit_pages_factory do
     %{
       table: "imported_exit_pages",
-      date: @today,
+      date: Timex.today(),
       exit_page: "",
       visitors: 1,
       exits: 1
@@ -242,7 +240,7 @@ defmodule Plausible.Factory do
   def imported_locations_factory do
     %{
       table: "imported_locations",
-      date: @today,
+      date: Timex.today(),
       country: "",
       region: "",
       city: 0,
@@ -256,7 +254,7 @@ defmodule Plausible.Factory do
   def imported_devices_factory do
     %{
       table: "imported_devices",
-      date: @today,
+      date: Timex.today(),
       device: "",
       visitors: 1,
       visits: 1,
@@ -268,7 +266,7 @@ defmodule Plausible.Factory do
   def imported_browsers_factory do
     %{
       table: "imported_browsers",
-      date: @today,
+      date: Timex.today(),
       browser: "",
       visitors: 1,
       visits: 1,
@@ -280,7 +278,7 @@ defmodule Plausible.Factory do
   def imported_operating_systems_factory do
     %{
       table: "imported_operating_systems",
-      date: @today,
+      date: Timex.today(),
       operating_system: "",
       visitors: 1,
       visits: 1,

test/support/test_utils.ex

@@ -14,7 +14,7 @@ defmodule Plausible.TestUtils do
   def add_imported_data(%{site: site}) do
     site =
       site
-      |> Plausible.Site.set_imported_source("Google Analytics")
+      |> Plausible.Site.start_import("Google Analytics", "ok")
       |> Repo.update!()
 
     {:ok, site: site}