From ce238399321948fcf09641cf7cb99c9fce6fee60 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 2 Apr 2021 11:42:44 +0300 Subject: [PATCH 1/2] Use DBIP database for countries --- .gitignore | 4 +- CHANGELOG.md | 1 + assets/js/dashboard/mount.js | 3 +- assets/js/dashboard/stats/countries.js | 11 +++++- config/runtime.exs | 2 +- config/test.exs | 9 ----- lib/mix/tasks/download_country_database.ex | 38 +++++++++++++++++++ .../templates/stats/stats.html.eex | 2 +- .../api/external_controller_test.exs | 5 +-- 9 files changed, 57 insertions(+), 18 deletions(-) create mode 100644 lib/mix/tasks/download_country_database.ex diff --git a/.gitignore b/.gitignore index 9a60ff83f53f..3f5eff64fa5b 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,7 @@ npm-debug.log # we ignore priv/static. You may want to comment # this depending on your deployment strategy. /priv/static/ -/priv/geolix/ +/priv/geodb/ # Files matching config/*.secret.exs pattern contain sensitive # data and you should not commit them into version control. @@ -58,4 +58,4 @@ plausible-report.xml # Dializer /priv/plts/*.plt -/priv/plts/*.plt.hash \ No newline at end of file +/priv/plts/*.plt.hash diff --git a/CHANGELOG.md b/CHANGELOG.md index 8586258b3ae5..63f59d0d61de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - Site switching keybinds (1-9 for respective sites) plausible/analytics#735 - Glob (wildcard) based pageview goals plausible/analytics#750 - Support for embedding shared links in an iframe plausible/analytics#812 +- Include a basic IP-To-Country database by default plausible/analytics#906 ### Fixed - Capitalized date/time selection keybinds not working plausible/analytics#709 diff --git a/assets/js/dashboard/mount.js b/assets/js/dashboard/mount.js index c42dbc199638..db86a5b18118 100644 --- a/assets/js/dashboard/mount.js +++ b/assets/js/dashboard/mount.js @@ -15,7 +15,8 @@ if (container) { hasGoals: container.dataset.hasGoals === 'true', insertedAt: container.dataset.insertedAt, embedded: container.dataset.embedded, - background: container.dataset.background + background: container.dataset.background, + selfhosted: container.dataset.selfhosted === 'true' } const loggedIn = container.dataset.loggedIn === 'true' diff --git a/assets/js/dashboard/stats/countries.js b/assets/js/dashboard/stats/countries.js index 576a62faca48..20dd363d0af2 100644 --- a/assets/js/dashboard/stats/countries.js +++ b/assets/js/dashboard/stats/countries.js @@ -125,13 +125,22 @@ class Countries extends React.Component { }); } + geolocationDbNotice() { + if (this.props.site.selfhosted) { + return ( + IP Geolocation by DB-IP + ) + } + } + renderBody() { if (this.state.countries) { return (

Countries

-
+
+ { this.geolocationDbNotice() } ) } diff --git a/config/runtime.exs b/config/runtime.exs index bdd58fef9681..c0b065c19d86 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -56,7 +56,7 @@ cron_enabled = String.to_existing_atom(System.get_env("CRON_ENABLED", "false")) custom_domain_server_ip = System.get_env("CUSTOM_DOMAIN_SERVER_IP") custom_domain_server_user = System.get_env("CUSTOM_DOMAIN_SERVER_USER") custom_domain_server_password = System.get_env("CUSTOM_DOMAIN_SERVER_PASSWORD") -geolite2_country_db = System.get_env("GEOLITE2_COUNTRY_DB") +geolite2_country_db = System.get_env("GEOLITE2_COUNTRY_DB", "priv/geodb/dbip-country.mmdb") disable_auth = String.to_existing_atom(System.get_env("DISABLE_AUTH", "false")) disable_registration = String.to_existing_atom(System.get_env("DISABLE_REGISTRATION", "false")) hcaptcha_sitekey = System.get_env("HCAPTCHA_SITEKEY") diff --git a/config/test.exs b/config/test.exs index 3d3b6d9a7de9..5d75db8deaf5 100644 --- a/config/test.exs +++ b/config/test.exs @@ -18,14 +18,5 @@ config :plausible, paddle_api: Plausible.PaddleApi.Mock, google_api: Plausible.Google.Api.Mock -config :geolix, - databases: [ - %{ - id: :country, - adapter: Geolix.Adapter.Fake, - data: %{{1, 1, 1, 1} => %{country: %{iso_code: "US"}}} - } - ] - config :plausible, session_timeout: 0 diff --git a/lib/mix/tasks/download_country_database.ex b/lib/mix/tasks/download_country_database.ex new file mode 100644 index 000000000000..d6e63e34fd88 --- /dev/null +++ b/lib/mix/tasks/download_country_database.ex @@ -0,0 +1,38 @@ +defmodule Mix.Tasks.DownloadCountryDatabase do + use Mix.Task + use Plausible.Repo + require Logger + + # coveralls-ignore-start + + def run(_) do + Application.ensure_all_started(:httpoison) + Application.ensure_all_started(:timex) + this_month = Timex.today() + last_month = Timex.shift(this_month, months: -1) + this_month = this_month |> Date.to_iso8601() |> binary_part(0, 7) + last_month = last_month |> Date.to_iso8601() |> binary_part(0, 7) + this_month_url = "https://download.db-ip.com/free/dbip-country-lite-#{this_month}.mmdb.gz" + last_month_url = "https://download.db-ip.com/free/dbip-country-lite-#{last_month}.mmdb.gz" + Logger.info("Downloading #{this_month_url}") + res = HTTPoison.get!(this_month_url) + + res = + case res.status_code do + 404 -> + Logger.info("Got 404 for #{this_month_url}, trying #{last_month_url}") + HTTPoison.get!(last_month_url) + + _ -> + res + end + + if res.status_code == 200 do + File.mkdir("priv/geodb") + File.write!("priv/geodb/dbip-country.mmdb", res.body) + Logger.info("Downloaded and saved the database successfully") + else + Logger.error("Unable to download and save the database. Response: #{inspect(res)}") + end + end +end diff --git a/lib/plausible_web/templates/stats/stats.html.eex b/lib/plausible_web/templates/stats/stats.html.eex index 8fd38ad8fa44..f6a9212f1e7b 100644 --- a/lib/plausible_web/templates/stats/stats.html.eex +++ b/lib/plausible_web/templates/stats/stats.html.eex @@ -5,7 +5,7 @@ <% end %>
-
+
<%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %>
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index 6cae5141eb4c..41ec87166a5a 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -454,7 +454,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert event.referrer == "" end - # Fake data is set up in config/test.exs test "looks up the country from the ip address", %{conn: conn} do params = %{ name: "pageview", @@ -464,12 +463,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") - |> put_req_header("x-forwarded-for", "1.1.1.1") + |> put_req_header("x-forwarded-for", "78.76.58.149") |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-20.com") - assert pageview.country_code == "US" + assert pageview.country_code == "SE" end test "URL is decoded", %{conn: conn} do From bfbadd16e9586759ce8f6c0ad0b6e47c8d11c42e Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Mon, 12 Apr 2021 12:16:39 +0300 Subject: [PATCH 2/2] Add fake data for geolocation tests --- config/runtime.exs | 4 ++-- config/test.exs | 9 +++++++++ .../controllers/api/external_controller_test.exs | 5 +++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index c0b065c19d86..57cdfeec7766 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -56,7 +56,7 @@ cron_enabled = String.to_existing_atom(System.get_env("CRON_ENABLED", "false")) custom_domain_server_ip = System.get_env("CUSTOM_DOMAIN_SERVER_IP") custom_domain_server_user = System.get_env("CUSTOM_DOMAIN_SERVER_USER") custom_domain_server_password = System.get_env("CUSTOM_DOMAIN_SERVER_PASSWORD") -geolite2_country_db = System.get_env("GEOLITE2_COUNTRY_DB", "priv/geodb/dbip-country.mmdb") +geolite2_country_db = System.get_env("EOLITE2_COUNTRY_DB", "priv/geodb/dbip-country.mmdb") disable_auth = String.to_existing_atom(System.get_env("DISABLE_AUTH", "false")) disable_registration = String.to_existing_atom(System.get_env("DISABLE_REGISTRATION", "false")) hcaptcha_sitekey = System.get_env("HCAPTCHA_SITEKEY") @@ -243,7 +243,7 @@ config :kaffy, ] ] -if geolite2_country_db do +if config_env() != :test && geolite2_country_db do config :geolix, databases: [ %{ diff --git a/config/test.exs b/config/test.exs index 5d75db8deaf5..3d3b6d9a7de9 100644 --- a/config/test.exs +++ b/config/test.exs @@ -18,5 +18,14 @@ config :plausible, paddle_api: Plausible.PaddleApi.Mock, google_api: Plausible.Google.Api.Mock +config :geolix, + databases: [ + %{ + id: :country, + adapter: Geolix.Adapter.Fake, + data: %{{1, 1, 1, 1} => %{country: %{iso_code: "US"}}} + } + ] + config :plausible, session_timeout: 0 diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index 41ec87166a5a..6cae5141eb4c 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -454,6 +454,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert event.referrer == "" end + # Fake data is set up in config/test.exs test "looks up the country from the ip address", %{conn: conn} do params = %{ name: "pageview", @@ -463,12 +464,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") - |> put_req_header("x-forwarded-for", "78.76.58.149") + |> put_req_header("x-forwarded-for", "1.1.1.1") |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-20.com") - assert pageview.country_code == "SE" + assert pageview.country_code == "US" end test "URL is decoded", %{conn: conn} do