Merge branch master of https://github.com/PostHog/posthog into handle…

…-topic-offsets

.all-contributorsrc

@@ -1926,6 +1926,42 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "hakubo",
+      "name": "Jakub Olek",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1018759?v=4",
+      "profile": "https://github.com/hakubo",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "klink",
+      "name": "Kate Klink",
+      "avatar_url": "https://avatars.githubusercontent.com/u/5008686?v=4",
+      "profile": "http://kateklink.com",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "rafalmierzwiak",
+      "name": "Rafal Mierzwiak",
+      "avatar_url": "https://avatars.githubusercontent.com/u/11348586?v=4",
+      "profile": "https://www.rm.pl",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "gitstart",
+      "name": "GitStart",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1501599?v=4",
+      "profile": "http://gitstart.github.io/",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.devcontainer/Dockerfile

@@ -7,7 +7,7 @@
 # experience as rich as possible. Perhaps later down the line it might be worth
 # rolling our own
 #
-FROM mcr.microsoft.com/vscode/devcontainers/python:3.9-bullseye
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.8-bullseye
 
 # Make sure all exit codes on pipes cause failures
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -49,3 +49,4 @@ WORKDIR /workspace
 COPY requirements.txt requirements-dev.txt ./
 RUN pip install -r requirements-dev.txt --compile --no-cache-dir && \
     pip install -r requirements.txt --compile --no-cache-dir
+

.devcontainer/devcontainer.json

@@ -7,13 +7,32 @@
     "build": {
         "dockerfile": "Dockerfile",
         "context": "../",
-        "cacheFrom": ""
+        "cacheFrom": "ghcr.io/posthog/posthog/codespaces:master"
     },
 
     "remoteUser": "vscode",
+    "remoteEnv": {
+        "DEBUG": "1",
+        "DATABASE_URL": "postgres://posthog:posthog@localhost:5432/posthog",
+        "KAFKA_ENABLED": "true",
+        "KAFKA_HOSTS": "kafka:9092",
+        "KAFKA_URL": "kafka://kafka:9092",
+        "CLICKHOUST_HOST": "localhost",
+        "CLICKHOUSE_DATABASE": "posthog_test",
+        "CLICKHOUSE_VERIFY": "False",
+        "REDIS_URL": "redis://localhost:6379"
+    },
     "overrideCommand": false,
     "mounts": ["source=codespaces-linux-var-lib-docker,target=/var/lib/docker,type=volume"],
-    "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined", "--privileged", "--init"],
+    "runArgs": [
+        "--cap-add=SYS_PTRACE",
+        "--security-opt",
+        "seccomp=unconfined",
+        "--privileged",
+        "--init",
+        "--add-host",
+        "kafka:127.0.0.1"
+    ],
 
     "workspaceFolder": "/workspace",
 

.dockerignore

@@ -29,3 +29,4 @@
 !plugin-server/src
 !plugin-server/.eslintrc.js
 !plugin-server/.prettierrc
+!share/GeoLite2-City.mmdb

.github/ISSUE_TEMPLATE/sprint_planning_retro.md

@@ -7,6 +7,12 @@ title: Sprint 1.n.0 m/2 - Jan 1 to Jan 12
 
 # Global Sprint Planning
 
+## 3 things that might take us down
+
+1.
+2.
+3.
+
 ## Retro: Status of Outcomes from Previous Sprint
 
 1.

.github/actions/run-backend-tests/action.yml

@@ -75,8 +75,8 @@ runs:
               mkdir -p frontend/dist
               touch frontend/dist/index.html
               touch frontend/dist/layout.html
-              touch frontend/dist/shared_dashboard.html
               touch frontend/dist/exporter.html
+              [ ! -f ./share/GeoLite2-City.mmdb ] && ( curl -L "https://mmdbcdn.posthog.net/" | brotli --decompress --output=./share/GeoLite2-City.mmdb )
 
         - name: Wait for Clickhouse & Kafka
           shell: bash

.github/workflows/build-and-deploy-prod.yml

@@ -20,8 +20,6 @@ jobs:
             contents: read
             id-token: write
         steps:
-            - name: Set up Depot CLI
-              uses: depot/setup-action@v1
             - name: Configure AWS credentials
               uses: aws-actions/configure-aws-credentials@v1
               with:
@@ -46,24 +44,14 @@ jobs:
                   # ref: 'master'
                   path: 'deploy/'
 
-            - name: Build image
-              uses: depot/build-push-action@v1
-              with:
-                  context: .
-                  file: prod.web.Dockerfile
-                  load: true
-                  tags: |
-                      ${{ steps.login-ecr.outputs.registry }}/posthog-production:${{ github.sha }}
-                      ${{ steps.login-ecr.outputs.registry }}/posthog-production:latest
-                  project: 1stsk4xt19 # posthog-cloud project
-
-            - name: Push image to Amazon ECR
+            - name: Build, tag, and push image to Amazon ECR
               id: build-image
               env:
                   ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
                   ECR_REPOSITORY: posthog-production
                   IMAGE_TAG: ${{ github.sha }}
               run: |
+                  docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:latest -f prod.web.Dockerfile .
                   docker push --all-tags $ECR_REGISTRY/$ECR_REPOSITORY
                   echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"
 
@@ -213,27 +201,27 @@ jobs:
                         "context": ${{ toJson(github) }}
                       }
 
-    # TODO: Bring back once https://github.com/rtCamp/action-slack-notify/issues/126 is resolved
-    # slack:
-    #     name: Notify Slack of start of deploy
-    #     runs-on: ubuntu-20.04
-    #     if: github.repository == 'posthog/posthog'
-    #     steps:
-    #         - name: Notify Platform team on slack
-    #           uses: rtCamp/action-slack-notify@v2
-    #           env:
-    #               SLACK_CHANNEL: platform-bots
-    #               SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff'
-    #               SLACK_ICON: https://github.com/posthog.png?size=48
-    #               SLACK_MESSAGE: 'Production Cloud Deploy Beginning :rocket: - ${{ github.event.head_commit.message }}'
-    #               SLACK_TITLE: Message
-    #               SLACK_USERNAME: Max Hedgehog
-    #               SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+    slack:
+        name: Notify Slack of start of deploy
+        runs-on: ubuntu-20.04
+        if: github.repository == 'posthog/posthog'
+        steps:
+            - name: Notify Platform team on slack
+              uses: rtCamp/action-slack-notify@v2
+              env:
+                  SLACK_CHANNEL: platform-bots
+                  SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff'
+                  SLACK_ICON: https://github.com/posthog.png?size=48
+                  SLACK_MESSAGE: 'Production Cloud Deploy Beginning :rocket: - ${{ github.event.head_commit.message }}'
+                  SLACK_TITLE: Message
+                  SLACK_USERNAME: Max Hedgehog
+                  SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
 
     sentry:
         name: Notify Sentry of a production release
         runs-on: ubuntu-20.04
         if: github.repository == 'PostHog/posthog'
+        needs: build
         steps:
             - name: Checkout master
               uses: actions/checkout@v2

.github/workflows/ci-async-migrations.yml

@@ -71,7 +71,6 @@ jobs:
                   mkdir -p frontend/dist
                   touch frontend/dist/index.html
                   touch frontend/dist/layout.html
-                  touch frontend/dist/shared_dashboard.html
                   touch frontend/dist/exporter.html
 
             - name: Wait for Clickhouse & Kafka

.github/workflows/ci-backend.yml

@@ -398,7 +398,6 @@ jobs:
                   python manage.py collectstatic --noinput
                   touch frontend/dist/index.html
                   touch frontend/dist/layout.html
-                  touch frontend/dist/shared_dashboard.html
                   touch frontend/dist/exporter.html
 
             - name: Run cloud tests (posthog-cloud)

.github/workflows/lint-pr.yml

@@ -15,6 +15,3 @@ jobs:
             - uses: amannn/action-semantic-pull-request@v4
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-              with:
-                  validateSingleCommit: true
-                  validateSingleCommitMatchesPrTitle: true

.gitignore

@@ -41,3 +41,4 @@ ee/benchmarks/results
 coverage-*.xml
 object_storage/
 __emails__
+share/GeoLite2-City.mmdb

bin/docker-server

@@ -3,6 +3,13 @@ set -e
 
 ./bin/migrate-check
 
+# To ensure we are able to expose metrics from multiple processes, we need to
+# provide a directory for `prometheus_client` to store a shared registry.
+export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d)
+trap 'rm -rf "$PROMETHEUS_MULTIPROC_DIR"' EXIT
+
+export PROMETHEUS_METRICS_EXPORT_PORT=8001
+
 gunicorn posthog.wsgi \
     --config gunicorn.config.py \
     --bind 0.0.0.0:8000 \
@@ -13,4 +20,4 @@ gunicorn posthog.wsgi \
     --workers=2 \
     --threads=4 \
     --worker-class=gthread \
-    --limit-request-line=8190
+    --limit-request-line=8190 $@

bin/docker-worker

@@ -3,5 +3,12 @@ set -e
 
 ./bin/migrate-check
 
+# Stop any background jobs on exit
+trap 'kill $(jobs -p)' EXIT
+
 ./bin/plugin-server &
-./bin/docker-worker-celery --with-scheduler
+./bin/docker-worker-celery --with-scheduler &
+
+# Exit if any processes exit, and exit with it's exit code
+wait -n
+exit $?

bin/docker-worker-celery

@@ -59,6 +59,9 @@ while test $# -gt 0; do
   esac
 done
 
+# Stop any background jobs on exit
+trap 'kill $(jobs -p)' EXIT
+
 if [ "$with_scheduler" == "true" ]; then
   ./bin/docker-worker-beat &
 fi
@@ -80,7 +83,9 @@ echo
 
 ./bin/migrate-check
 
-SKIP_ASYNC_MIGRATIONS_SETUP=0 celery -A posthog worker ${FLAGS[*]}
+SKIP_ASYNC_MIGRATIONS_SETUP=0 celery -A posthog worker ${FLAGS[*]} &
+
+# Exit if any processes exit, and exit with it's exit code
+wait -n
+exit $?
 
-# Stop the beat!
-trap 'kill $(jobs -p)' EXIT

bin/plugin-server

@@ -44,7 +44,7 @@ fi
 
 if [[ -n $NO_RESTART_LOOP ]]; then
   echo "▶️ Starting plugin server..."
-  trap 'kill -TERM $child 2>/dev/null; while kill -0 $child 2>/dev/null; do sleep 1; done' SIGTERM
+  trap 'kill -TERM $child 2>/dev/null; while kill -0 $child 2>/dev/null; do sleep 1; done' EXIT
   yarn $cmd &
   child=$!
   wait $child

bin/start

@@ -7,20 +7,15 @@ trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
 export DEBUG=${DEBUG:-1}
 export SKIP_SERVICE_VERSION_REQUIREMENTS=1
 
-ARCH=$(uname -m)
-if [ "$ARCH" == "arm64" ]; then
-  DOCKER_COMPOSE_VARIANT='arm64'
-else
-  DOCKER_COMPOSE_VARIANT='dev'
-fi
-
 service_warning() {
-  echo -e "\033[0;31m$1 isn't ready. You can run the stack with:\ndocker compose -f docker-compose.${DOCKER_COMPOSE_VARIANT}.yml up kafka clickhouse db redis\nIf you have already ran that, just make sure that services are starting properly, and sit back.\nWaiting for $1 to start...\033[0m"
+  echo -e "\033[0;31m$1 isn't ready. You can run the stack with:\ndocker compose -f docker-compose.dev.yml up\nIf you have already ran that, just make sure that services are starting properly, and sit back.\nWaiting for $1 to start...\033[0m"
 }
 
 nc -z localhost 9092 || ( service_warning 'Kafka'; bin/check_kafka_clickhouse_up )
 curl -s 'http://localhost:8123/ping' || ( service_warning 'ClickHouse'; bin/check_kafka_clickhouse_up )
 
+[ ! -f ./share/GeoLite2-City.mmdb ] && ( curl -L "https://mmdbcdn.posthog.net/" | brotli --decompress --output=./share/GeoLite2-City.mmdb )
+
 ./bin/start-worker &
 ./bin/start-backend &
 ./bin/start-frontend &