Move Complement test matrix jobs definition to match Sytest and Trial.

.ci/scripts/calculate_jobs.py

@@ -133,3 +133,63 @@ def set_output(key: str, value: str):
 
 test_matrix = json.dumps(sytest_tests)
 set_output("sytest_test_matrix", test_matrix)
+
+
+# First calculate the workers used during complement jobs
+#
+# Will only be used on tests with SYNAPSE_COMPLEMENT_USE_WORKERS enabled
+# Github sees a none existent key and evaluates it to an empty string, according to
+# https://docs.github.com/en/actions/learn-github-actions/contexts
+# See the last sentence before "Determining when to use contexts"
+
+complement_test_jobs = [
+    {"arrangement": "monolith", "database": "SQLite"},
+    {"arrangement": "monolith", "database": "Postgres"},
+    {
+        "arrangement": "workers",
+        "database": "Postgres",
+        "worker_types": ", ".join(
+            worker
+            for worker in [
+                "event_persister",
+                "event_persister",
+                "background_worker",
+                "frontend_proxy",
+                "event_creator",
+                "user_dir",
+                "media_repository",
+                "federation_inbound",
+                "federation_reader",
+                "federation_sender",
+                "synchrotron",
+                "client_reader",
+                "appservice",
+                "pusher",
+            ]
+        ),
+    },
+]
+
+# Testing for backwards compatibility mode. It's a simple matter of just NOT putting in
+# a worker_types string. It will resolve to an empty string on the other side.
+
+complement_backwards_compatibility_tests = [
+    {
+        "database": "Postgres",
+        "arrangement": "workers",
+    }
+]
+
+print("::group::Calculated complement job with workers")
+print(json.dumps(complement_test_jobs, indent=4))
+print("::endgroup::")
+
+print("::group::Calculated backwards compatibility complement job with workers")
+print(json.dumps(complement_backwards_compatibility_tests, indent=4))
+print("::endgroup::")
+
+test_matrix = json.dumps(complement_test_jobs)
+set_output("complement_test_matrix", test_matrix)
+
+test_matrix = json.dumps(complement_backwards_compatibility_tests)
+set_output("complement_backwards_compatibility_test_matrix", test_matrix)

.github/workflows/tests.yml

@@ -148,6 +148,8 @@ jobs:
     outputs:
       trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
       sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
+      complement_test_matrix: ${{ steps.get-matrix.outputs.complement_test_matrix }}
+      complement_backwards_compatibility_test_matrix: ${{ steps.get-matrix.outputs.complement_backwards_compatibility_test_matrix }}
 
   trial:
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
@@ -436,21 +438,50 @@ jobs:
 
   complement:
     if: "${{ !failure() && !cancelled() }}"
-    needs: linting-done
+    needs: calculate-test-jobs
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arrangement: monolith
-            database: SQLite
+        job: ${{ fromJson(needs.calculate-test-jobs.outputs.complement_test_matrix) }}
+
+    steps:
+      - name: Run actions/checkout@v3 for synapse
+        uses: actions/checkout@v3
+        with:
+          path: synapse
 
-          - arrangement: monolith
-            database: Postgres
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: 1.58.1
+            override: true
+      - uses: Swatinem/rust-cache@v2
 
-          - arrangement: workers
-            database: Postgres
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
+
+      - name: Run Complement Tests
+        shell: bash
+        env:
+          POSTGRES: ${{ (matrix.job.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.job.arrangement == 'workers') && 1 || '' }}
+          WORKER_TYPES: ${{ matrix.job.worker_types }}
+        run: |
+          set -o pipefail
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+
+  back-compat:
+    name: "Backwards Compatibility Test"
+    if: "${{ !failure() && !cancelled() }}"
+    needs: calculate-test-jobs
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        job: ${{ fromJson(needs.calculate-test-jobs.outputs.complement_backwards_compatibility_test_matrix) }}
 
     steps:
       - name: Run actions/checkout@v3 for synapse
@@ -468,11 +499,15 @@ jobs:
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
-      - run: |
-          set -o pipefail
-          POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+      - name: Run Complement Tests
         shell: bash
-        name: Run Complement Tests
+        env:
+          POSTGRES: ${{ (matrix.job.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.job.arrangement == 'workers') && 1 || '' }}
+          WORKER_TYPES: ${{ matrix.job.worker_types }}
+        run: |
+          set -o pipefail
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
 
   cargo-test:
     if: ${{ needs.changes.outputs.rust == 'true' }}

changelog.d/14153.misc

@@ -0,0 +1 @@
+Move Complement CI job matrix definition to central location to match Sytest and Trial. Contributed by Jason Little.

docker/complement/conf/start_for_complement.sh

@@ -6,7 +6,10 @@ set -e
 
 echo "Complement Synapse launcher"
 echo "  Args: $@"
-echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS"
+echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE"
+echo "  Env: SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS"
+echo "  Env: SYNAPSE_WORKER_TYPES=$SYNAPSE_WORKER_TYPES"
+echo "  NOTE: if using the backward compatibility mode, worker types aren't declared yet."
 
 function log {
     d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
@@ -45,7 +48,12 @@ esac
 
 if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
   # Specify the workers to test with
-  export SYNAPSE_WORKER_TYPES="\
+  # Allow overriding by explicitly setting SYNAPSE_WORKER_TYPES outside, while still
+  # utilizing WORKERS=1 for backwards compatibility.
+  # -n True if the length of string is non-zero.
+  # -z True if the length of string is zero.
+  if [[ -z "$SYNAPSE_WORKER_TYPES" ]]; then
+    export SYNAPSE_WORKER_TYPES="\
       event_persister, \
       event_persister, \
       background_worker, \
@@ -57,10 +65,11 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
       federation_reader, \
       federation_sender, \
       synchrotron, \
-      client_reader, \
       appservice, \
       pusher"
 
+    log "Workers now declared: $SYNAPSE_WORKER_TYPES"
+  fi
   # Improve startup times by using a launcher based on fork()
   export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1
 else

docs/development/contributing_guide.md

@@ -323,7 +323,8 @@ COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestImportHistoric
 The above will run a monolithic (single-process) Synapse with SQLite as the database. For other configurations, try:
 
 - Passing `POSTGRES=1` as an environment variable to use the Postgres database instead.
-- Passing `WORKERS=1` as an environment variable to use a workerised setup instead. This option implies the use of Postgres.
+- Passing `WORKERS=1` as an environment variable to use a set of workers that mirrors what is used in Sytest. This option implies the use of Postgres.
+  - If setting `WORKERS=1`, optionally set `WORKER_TYPES=` to declare which worker types you wish to test. A simple comma-delimited string containing the worker types defined from the template in [here](https://github.com/matrix-org/synapse/blob/develop/docker/configure_workers_and_start.py). A safe example would be `WORKER_TYPES="federation_inbound,federation_sender,synchrotron,"`. See the [worker documentation](https://matrix-org.github.io/synapse/latest/workers.html) for additional information on workers.
 
 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
 ```sh

scripts-dev/complement.sh

@@ -139,6 +139,9 @@ if [[ -n "$WORKERS" ]]; then
   # Use workers.
   export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true
 
+  # Pass through the workers defined. If none, it will be an empty string
+  export PASS_SYNAPSE_WORKER_TYPES="$WORKER_TYPES"
+
   # Workers can only use Postgres as a database.
   export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres