Merge branch 'v5.0.0-rc.0' into fastapi

* v5.0.0-rc.0:
  Add simple redirector
  Tidy up the layout of the analysis viewer
  Add an observations viewer
  Get rid of all the dataviz that isn't the analysis visualizer
  Release/5.0.0 alpha3 (#81)
  Offset analysis schedule by 6 hours
  Add support for temporal cloud (#79)
  fix: support for sorting network_events using transaction_id (#51)
  Optimize performance of table writers and refactor table model (#74)
  Improvements related to deployment (#69)
  Add .codecov file
  Update jsonl sync example
  Setup workflow to publish ooni data docs (#73)
  OONI Pipeline v5 alpha (#64)
  Fix codecov (#62)
  Temporal workflows (#61)
  OONI Data, OONI Pipeline split (#60)
  Add support for caching netinfodb

.codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        target: 80
+    patch:
+      default:
+        target: 80

.github/workflows/build_docs.yml

@@ -0,0 +1,50 @@
+name: build docs
+on: push
+
+jobs:
+  build_docs:
+    runs-on: "ubuntu-20.04"
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Build docs
+        run: make docs
+
+      - name: Get current git ref
+        id: rev_parse
+        run: echo "COMMIT_HASH=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Checkout ooni/docs
+        uses: actions/checkout@v2
+        with:
+          repository: "ooni/docs"
+          ssh-key: ${{ secrets.OONI_DOCS_DEPLOYKEY }}
+          path: "ooni-docs"
+
+      - name: Update docs
+        run: |
+          mkdir -p ooni-docs/src/content/docs/data/
+          cp -R dist/docs/* ooni-docs/src/content/docs/data/
+
+      - name: Check for conflicting slugs
+        run: |
+          cat ooni-docs/src/content/docs/data/*.md \
+            | grep "^slug:" | awk -F':' '{gsub(/^ +/, "", $2); print $2}' | sort | uniq -c \
+            | awk '{if ($1 > 1) { print "duplicate slug for: " $2; exit 1}}'
+
+      - name: Print the lines of the generated docs
+        run: wc -l ooni-docs/src/content/docs/data/*
+
+      - name: Commit changes
+        # Only push the docs update when we are in master
+        # if: github.ref == 'refs/heads/master'
+        run: |
+          cd ooni-docs
+          git config --global user.email "[email protected]"
+          git config --global user.name "OONI Github Actions Bot"
+          git add .
+          git commit -m "auto: update ooni/data docs to ${{ steps.rev_parse.outputs.COMMIT_HASH }}" || echo "No changes to commit"
+          git push origin

.github/workflows/test_oonidata.yml

@@ -0,0 +1,50 @@
+name: test oonidata
+on: push
+jobs:
+  run_tests:
+    name: ${{ matrix.os }} / ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      matrix:
+        os: [Ubuntu]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Set up datadir cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/datadir/
+          key: oonidata-tests-data-datadir
+
+      - name: Set up measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/measurements/
+          key: oonidata-tests-data-measurements
+
+      - name: Set up raw_measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/raw_measurements/
+          key: oonidata-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }}
+
+      - name: Run all tests
+        run: hatch run cov
+        working-directory: ./oonidata/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: oonidata
+          working-directory: ./oonidata/
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/test_oonipipeline.yml

@@ -0,0 +1,62 @@
+name: test oonipipeline
+on: push
+jobs:
+  run_tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+
+      # Contrary to oonidata, which is a tool for end users, oonipipeline
+      # targets a single python version to minimize the likelyhood of breakage
+      # in production.
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Set up datadir cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/datadir/
+          key: oonipipeline-tests-data-datadir
+
+      - name: Set up measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/measurements/
+          key: oonipipeline-tests-data-measurements
+
+      - name: Set up raw_measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/raw_measurements/
+          key: oonipipeline-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }}
+
+      - name: Install clickhouse
+        run: |
+          sudo apt-get install -y apt-transport-https ca-certificates dirmngr
+          sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
+          echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
+              /etc/apt/sources.list.d/clickhouse.list
+          sudo apt-get update
+          sudo apt-get install -y clickhouse-server clickhouse-client
+
+      - name: Install temporal
+        run: |
+          curl -sSf https://temporal.download/cli.sh | sh
+          echo "$HOME/.temporalio/bin" >> $GITHUB_PATH
+
+      - name: Run all tests
+        run: hatch run cov -v
+        working-directory: ./oonipipeline/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: oonipipeline
+          working-directory: ./oonipipeline/
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -1,12 +1,13 @@
 __pycache__
-/.coverage
-/coverage.xml
-/tests/data/datadir/*
-/tests/data/raw_measurements/*
-/tests/data/measurements/*
+.coverage
+.coverage.*
+coverage.xml
+
+.ipynb_checkpoints/
+
 /dist
-/datadir
 /output
 /attic
 /prof
-.ipynb_checkpoints/
+
+.DS_Store

.vscode/settings.json

@@ -3,6 +3,7 @@
   "python.testing.pytestArgs": ["tests"],
   "python.testing.unittestEnabled": false,
   "python.testing.pytestEnabled": true,
+  "python.defaultInterpreterPath": "${workspaceFolder}/oonipipeline/.venv",
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
   },

Makefile

@@ -0,0 +1,7 @@
+docs:
+	./scripts/build_docs.sh
+
+clean:
+	rm -rf dist/
+
+.PHONY: docs clean