Merge branch 'master' into support-postgresql-search_path

.dockerignore

@@ -0,0 +1,73 @@
+# This file excludes paths from the Docker build context.
+#
+# By default, Docker's build context includes all files (and folders) in the
+# current directory. Even if a file isn't copied into the container it is still sent to
+# the Docker daemon.
+#
+# There are multiple reasons to exclude files from the build context:
+#
+# 1. Prevent nested folders from being copied into the container (ex: exclude
+#    /assets/node_modules when copying /assets)
+# 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc)
+# 3. Avoid sending files containing sensitive information
+#
+# More information on using .dockerignore is available here:
+# https://docs.docker.com/engine/reference/builder/#dockerignore-file
+
+.dockerignore
+
+# Ignore git, but keep git HEAD and refs to access current commit hash if needed:
+#
+# $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat
+# d0b8727759e1e0e7aa3d41707d12376e373d5ecc
+.git
+!.git/HEAD
+!.git/refs
+
+# Common development/test artifacts
+/cover/
+/doc/
+/test/
+/tmp/
+.elixir_ls
+plausible-report.xml
+.env
+.idea
+*.iml
+*.log
+*.code-workspace
+.vscode
+
+# Mix artifacts
+/_build/
+/deps/
+*.ez
+
+# Generated on crash by the VM
+erl_crash.dump
+
+# If NPM crashes, it generates a log, let's ignore it too.
+npm-debug.log
+
+# Static artifacts - These should be fetched and built inside the Docker image
+/assets/node_modules/
+/tracker/node_modules/
+/priv/static/cache_manifest.json
+/priv/static/css
+/priv/static/js
+/priv/version.json
+
+# Auto-generated tracker files
+/priv/tracker/js/*.js
+
+# Dializer
+/priv/plts/*.plt
+/priv/plts/*.plt.hash
+
+# Geolocation databases
+/priv/geodb/*.mmdb
+/priv/geodb/*.mmdb.gz
+
+# Docker volumes
+.clickhouse_db_vol*
+plausible_db*

.github/workflows/build-private-images-ghcr.yml

@@ -0,0 +1,67 @@
+name: Build Private Images GHCR
+
+on:
+  push:
+    branches: [master, stable]
+    tags: ['r*']
+  pull_request:
+    types: [synchronize, labeled]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'deploy-to-staging') }}
+    runs-on: buildjet-16vcpu-ubuntu-2204
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Docker meta
+        id: meta
+        uses: docker/[email protected]
+        with:
+          images: ghcr.io/plausible/analytics/ee
+          tags: |
+            type=ref,event=pr
+            type=ref,event=branch
+            type=ref,event=tag
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            MIX_ENV=prod
+            BUILD_METADATA=${{ steps.meta.outputs.json }}
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
+
+      - name: Notify team on failure
+        if: ${{ failure() }}
+        uses: fjogeleit/http-request-action@v1
+        with:
+          url: ${{ secrets.BUILD_NOTIFICATION_URL }}
+          method: 'POST'
+          customHeaders: '{"Content-Type": "application/json"}'
+          data: '{"content": "<a href=\"https://github.com/plausible/analytics/actions/workflows/build-private-images.yml\">Build failed</a>"}'

.github/workflows/build-public-images-ghcr.yml

@@ -51,7 +51,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           build-args: |
-            MIX_ENV=small
+            MIX_ENV=ce
             BUILD_METADATA=${{ steps.meta.outputs.json }}
             ERL_FLAGS=+JPperf true
 

.github/workflows/build-public-images.yml

@@ -46,9 +46,9 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           build-args: |
-            MIX_ENV=small
+            MIX_ENV=ce
             BUILD_METADATA=${{ steps.meta.outputs.json }}
-            ERL_FLAGS=+JPperf true
+            ERL_FLAGS=+JMsingle true
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/elixir.yml

@@ -1,26 +1,42 @@
 name: Elixir CI
 
 on:
+  pull_request:
   push:
     branches: [master, stable]
-  pull_request:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  CACHE_VERSION: v7
+  PERSISTENT_CACHE_DIR: cached
+
 jobs:
   build:
-    name: Build and test
+    name: "Build and test (${{ matrix.mix_env }}, ${{ matrix.postgres_image }}${{ matrix.test_experimental_reduced_joins == '1' && ', experimental_reduced_joins' || '' }})"
     runs-on: ubuntu-latest
-
     strategy:
       matrix:
-        postgres_version: [13, 14, 15]
+        mix_env: ["test", "ce_test"]
+        postgres_image: ["postgres:16"]
+        test_experimental_reduced_joins: ["0"]
+
+        include:
+          - mix_env: "test"
+            postgres_image: "postgres:15"
+            test_experimental_reduced_joins: "0"
+          - mix_env: "test"
+            postgres_image: "postgres:16"
+            test_experimental_reduced_joins: "1"
 
+    env:
+      MIX_ENV: ${{ matrix.mix_env }}
+      TEST_EXPERIMENTAL_REDUCED_JOINS: ${{ matrix.test_experimental_reduced_joins }}
     services:
       postgres:
-        image: postgres:${{ matrix.postgres_version }}
+        image: ${{ matrix.postgres_image }}
         ports:
           - 5432:5432
         env:
@@ -30,7 +46,6 @@ jobs:
             --health-interval 10s
             --health-timeout 5s
             --health-retries 5
-
       clickhouse:
         image: clickhouse/clickhouse-server:23.3.7.5-alpine
         ports:
@@ -41,43 +56,96 @@ jobs:
             --health-interval 10s
             --health-timeout 5s
             --health-retries 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: marocchino/tool-versions-action@v1
+        id: versions
+      - uses: erlef/setup-beam@v1
+        with:
+          elixir-version: ${{ steps.versions.outputs.elixir }}
+          otp-version: ${{ steps.versions.outputs.erlang }}
+
+      - uses: actions/cache@v4
+        with:
+          path: |
+            deps
+            _build
+            tracker/node_modules
+            priv/tracker/js
+            ${{ env.PERSISTENT_CACHE_DIR }}
+          key: ${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: |
+            ${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-
+            ${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-refs/heads/master-
 
+      - name: Check for changes in tracker/**
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          filters: |
+            tracker:
+              - 'tracker/**'
+      - name: Check if priv/tracker/js/plausible.js exists
+        run: |
+          if [ -f priv/tracker/js/plausible.js ]; then
+            echo "HAS_BUILT_TRACKER=true" >> $GITHUB_ENV
+          else
+            echo "HAS_BUILT_TRACKER=false" >> $GITHUB_ENV
+          fi
+      - run: npm install --prefix ./tracker
+        if: steps.changes.outputs.tracker == 'true' || env.HAS_BUILT_TRACKER == 'false'
+      - run: npm run deploy --prefix ./tracker
+        if: steps.changes.outputs.tracker == 'true' || env.HAS_BUILT_TRACKER == 'false'
+
+      - run: mix deps.get --only $MIX_ENV
+      - run: mix compile --warnings-as-errors --all-warnings
+      - run: mix do ecto.create, ecto.migrate
+      - run: mix run -e "Tzdata.ReleaseUpdater.poll_for_update"
+
+      - run: make minio
+        if: env.MIX_ENV == 'test'
+      - run: mix test --include slow --include minio --max-failures 1 --warnings-as-errors
+        if: env.MIX_ENV == 'test'
+        env:
+          MINIO_HOST_FOR_CLICKHOUSE: "172.17.0.1"
+
+      - run: mix test --include slow --max-failures 1 --warnings-as-errors
+        if: env.MIX_ENV == 'ce_test'
+
+  static:
+    name: Static checks (format, credo, dialyzer)
+    env:
+      MIX_ENV: test
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Read .tool-versions
-        uses: marocchino/tool-versions-action@v1
+
+      - uses: marocchino/tool-versions-action@v1
         id: versions
-      - name: Set up Elixir
-        uses: erlef/setup-beam@v1
+      - uses: erlef/setup-beam@v1
         with:
-          elixir-version: ${{steps.versions.outputs.elixir}}
-          otp-version: ${{ steps.versions.outputs.erlang}}
-      - name: Restore dependencies cache
-        uses: buildjet/cache@v4
+          elixir-version: ${{ steps.versions.outputs.elixir }}
+          otp-version: ${{ steps.versions.outputs.erlang }}
+
+      - uses: actions/cache@v4
         with:
           path: |
             deps
             _build
             priv/plts
-          key: ${{ runner.os }}-mix-v7-${{ hashFiles('**/mix.lock') }}
-          restore-keys: ${{ runner.os }}-mix-v7-
-      - name: Install dependencies
-        run: mix deps.get && npm install --prefix ./tracker
-      - name: Check Formatting
-        run: mix format --check-formatted
-      - name: Check Compile Warnings
-        run: mix compile --warnings-as-errors --all-warnings
-      - name: Generate tracker files
-        run: npm run deploy --prefix ./tracker
-      - name: Check Credo Warnings
-        run: mix credo diff --from-git-merge-base origin/master
-      - name: Run tests
-        run: mix test --include slow --max-failures 1 --warnings-as-errors
-      - name: Run tests (small build)
-        run: MIX_ENV=small_test mix test --include slow --max-failures 1 --warnings-as-errors
-      - name: Check Dialyzer
-        run: mix dialyzer
-env:
-  MIX_ENV: test
+          key: static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: |
+            static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-
+            static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-refs/heads/master-
+
+      - run: mix deps.get
+      - run: mix compile --warnings-as-errors --all-warnings
+      - run: mix format --check-formatted
+      - run: mix deps.unlock --check-unused
+      - run: mix credo diff --from-git-merge-base origin/master
+      - run: mix dialyzer

.github/workflows/migrations-validation.yml

@@ -0,0 +1,28 @@
+name: Validate migrations
+
+on:
+  pull_request:
+    paths:
+      - 'priv/repo/migrations/**'
+      - 'priv/ingest_repo/migrations/**'
+
+jobs:
+  validate:
+    name: App code does not change at the same time
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: dorny/paths-filter@v3
+      id: changes
+      with:
+        list-files: json
+        filters: |
+          lib:
+            - 'lib/**'
+            - 'extra/**'
+            - 'config/**'
+
+    - if: steps.changes.outputs.lib == 'true'
+      run: |
+        echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time"
+        exit 1

.github/workflows/publish-docs.yml

@@ -51,7 +51,7 @@ jobs:
         run: mix docs
 
       - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./doc

.gitignore

@@ -19,6 +19,9 @@ erl_crash.dump
 # Also ignore archive artifacts (built via "mix archive.build").
 *.ez
 
+# Temporary files, for example, from tests.
+/tmp/
+
 # Ignore package tarball (built via "mix hex.build").
 plausible-*.tar