make robots.txt restrictive by default (#3905)

* make robots.txt restrictive by default

* add note about cloud robots.txt

CHANGELOG.md

@@ -32,6 +32,7 @@ All notable changes to this project will be documented in this file.
 - Add `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES` env var which defaults to `100000` (100KB)
 - Add alternative SMTP adapter plausible/analytics#3654
 - Add `EXTRA_CONFIG_PATH` env var to specify extra Elixir config plausible/analytics#3906
+- Add restrictive `robots.txt` for self-hosted plausible/analytics#3905
 
 ### Removed
 - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions

lib/plausible_web/endpoint.ex

@@ -33,11 +33,21 @@ defmodule PlausibleWeb.Endpoint do
   plug(PlausibleWeb.Tracker)
   plug(PlausibleWeb.Favicon)
 
+  static_paths = ~w(css js images favicon.ico)
+
+  static_paths =
+    on_full_build do
+      # NOTE: The Cloud uses custom robots.txt from https://github.com/plausible/website: https://plausible.io/robots.txt
+      static_paths
+    else
+      static_paths ++ ["robots.txt"]
+    end
+
   plug(Plug.Static,
     at: "/",
     from: :plausible,
     gzip: false,
-    only: ~w(css js images favicon.ico robots.txt)
+    only: static_paths
   )
 
   on_full_build do

priv/static/robots.txt

@@ -1,5 +1,2 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-agent: *
-# Disallow: /
+User-agent: *
+Disallow: /