From 469707c3c2bd20938d89a2f1a201d30ec172701f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 14:56:25 +0100
Subject: [PATCH 01/60] First attempt at a GHA pipeline

---
 .github/workflows/pipeline.yml | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 .github/workflows/pipeline.yml

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
new file mode 100644
index 000000000..85b9871db
--- /dev/null
+++ b/.github/workflows/pipeline.yml
@@ -0,0 +1,32 @@
+name: Run sytest
+on:
+  push:
+    branches: ["develop", "release-*"]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  sytest:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: bionic
+
+    container:
+      image: matrixdotorg/sytest-${{ matrix.sytest-tag }}
+      volumes:
+        - ${{ github.workspace }}:/src
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: TODO Prepare test blacklist
+      - name: Run bootstrap script
+        run: bash /bootstrap.sh synapse
+      - name: TODO upload artifacts
+      - name: TODO annotate logs

From 74bef07d6ea49898b2ba0bd7974e559bca43964d Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 15:09:46 +0100
Subject: [PATCH 02/60] Provide dummy "run"; parameterize homeserver impl

---
 .github/workflows/pipeline.yml | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 85b9871db..6732cd63f 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -16,6 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - homeserver-implementation: synapse
           - sytest-tag: bionic
 
     container:
@@ -26,7 +27,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: TODO Prepare test blacklist
-      - name: Run bootstrap script
-        run: bash /bootstrap.sh synapse
+        run: "false"
+      - name: Run sytest
+        run: bash /bootstrap.sh ${{ matrix.homeserver-implementation }}
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: "false"
+#        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: TODO upload artifacts
+        run: "false"
       - name: TODO annotate logs
+        run: "false"

From 4c86c1a4e7b3fd7a486379c9840255c106337b2d Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 15:15:50 +0100
Subject: [PATCH 03/60] Try to force GHA to trigger in my fork

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 6732cd63f..f80cdc07a 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -1,7 +1,7 @@
 name: Run sytest
 on:
   push:
-    branches: ["develop", "release-*"]
+    branches: ["develop", "release-*", "github-actions"]
   pull_request:
 
 concurrency:

From f68bbe4681c4e333b28565c58b77969ab31636b9 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 15:27:43 +0100
Subject: [PATCH 04/60] correct yaml nesting; fix image name

---
 .github/workflows/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index f80cdc07a..2322c1ea6 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -17,10 +17,10 @@ jobs:
       matrix:
         include:
           - homeserver-implementation: synapse
-          - sytest-tag: bionic
+            sytest-tag: bionic
 
     container:
-      image: matrixdotorg/sytest-${{ matrix.sytest-tag }}
+      image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}:${{ matrix.sytest-tag }}
       volumes:
         - ${{ github.workspace }}:/src
 

From 6f8197f6be66d5bef0fca607cc20af87062ceacc Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 15:43:37 +0100
Subject: [PATCH 05/60] Flesh out test cases and setup env

---
 .github/workflows/pipeline.yml | 51 ++++++++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 2322c1ea6..9ffba6808 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -16,24 +16,65 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - homeserver-implementation: synapse
+          - name: "Synapse - :python: 3.6 / SQLite / Monolith"
+            homeserver-implementation: synapse
             sytest-tag: bionic
 
+          - name: "Synapse - :python: 3.6 / :postgres: 10 / Monolith"
+            homeserver-implementation: synapse
+            sytest-tag: bionic
+            postgres: postgres
+
+          - name: "Synapse - :python: 3.6 / :postgres: 10 / Workers"
+            homeserver-implementation: synapse
+            sytest-tag: bionic
+            postgres: postgres
+            synapse-workers: workers
+
+          - name: "Synapse - :python: 3.9 / :postgres: 13 / Monolith"
+            homeserver-implementation: synapse
+            sytest-tag: testing
+            postgres: postgres
+
+          - name: "Synapse - :python: 3.9 / :postgres: 13 / Workers"
+            homeserver-implementation: synapse
+            sytest-tag: testing
+            postgres: postgres
+            synapse-workers: workers
+
+          - name: "Dendrite - :postgres:"
+            homeserver-implementation: dendrite
+            postgres: postgres
+
+          - name: "Dendrite - :postgres: / full HTTP APIs"
+            homeserver-implementation: dendrite
+            postgres: postgres
+            denrite-full-http: yes
+
+          - name: "Dendrite - :sqlite:"
+            homeserver-implementation: dendrite
+
+          - name: "Dendrite - :sqlite: / full HTTP APIs"
+            homeserver-implementation: dendrite
+            denrite-full-http: yes
+
     container:
       image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}:${{ matrix.sytest-tag }}
       volumes:
         - ${{ github.workspace }}:/src
+      env:
+        POSTGRES: ${{ matrix.postgres && 1 }}
+        WORKERS: ${{ matrix.synapse-workers && 1 }}
+        BLACKLIST: ${{ (matrix.synapse-workers && "synapse-blacklist-with-workers") || "synapse-blacklist" }}
+        API: ${{ matrix.dendrite-full-http && 1 }}
 
     steps:
       - uses: actions/checkout@v2
-      - name: TODO Prepare test blacklist
-        run: "false"
       - name: Run sytest
         run: bash /bootstrap.sh ${{ matrix.homeserver-implementation }}
       - name: Summarise results.tap
         if: ${{ always() }}
-        run: "false"
-#        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+        run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: TODO upload artifacts
         run: "false"
       - name: TODO annotate logs

From c4afc4f941dc4c0c9edbd90d45ace8f9c0a14276 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 15:49:33 +0100
Subject: [PATCH 06/60] GHA contexts require single quotes around strings

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 9ffba6808..3e65506d7 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -65,7 +65,7 @@ jobs:
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}
-        BLACKLIST: ${{ (matrix.synapse-workers && "synapse-blacklist-with-workers") || "synapse-blacklist" }}
+        BLACKLIST: ${{ (matrix.synapse-workers && 'synapse-blacklist-with-workers') || 'synapse-blacklist' }}
         API: ${{ matrix.dendrite-full-http && 1 }}
 
     steps:

From 40e9979d95662527402eb95a353706e61081c7f4 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 15:54:37 +0100
Subject: [PATCH 07/60] Names are redundant: don't use `yes`

---
 .github/workflows/pipeline.yml | 31 +++++++++++--------------------
 1 file changed, 11 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 3e65506d7..e48313b96 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -16,47 +16,38 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: "Synapse - :python: 3.6 / SQLite / Monolith"
-            homeserver-implementation: synapse
+          - homeserver-implementation: synapse
             sytest-tag: bionic
 
-          - name: "Synapse - :python: 3.6 / :postgres: 10 / Monolith"
-            homeserver-implementation: synapse
+          - homeserver-implementation: synapse
             sytest-tag: bionic
             postgres: postgres
 
-          - name: "Synapse - :python: 3.6 / :postgres: 10 / Workers"
-            homeserver-implementation: synapse
+          - homeserver-implementation: synapse
             sytest-tag: bionic
             postgres: postgres
             synapse-workers: workers
 
-          - name: "Synapse - :python: 3.9 / :postgres: 13 / Monolith"
-            homeserver-implementation: synapse
+          - homeserver-implementation: synapse
             sytest-tag: testing
             postgres: postgres
 
-          - name: "Synapse - :python: 3.9 / :postgres: 13 / Workers"
-            homeserver-implementation: synapse
+          - homeserver-implementation: synapse
             sytest-tag: testing
             postgres: postgres
             synapse-workers: workers
 
-          - name: "Dendrite - :postgres:"
-            homeserver-implementation: dendrite
+          - homeserver-implementation: dendrite
             postgres: postgres
 
-          - name: "Dendrite - :postgres: / full HTTP APIs"
-            homeserver-implementation: dendrite
+          - homeserver-implementation: dendrite
             postgres: postgres
-            denrite-full-http: yes
+            denrite-full-http: full-http
 
-          - name: "Dendrite - :sqlite:"
-            homeserver-implementation: dendrite
+          - homeserver-implementation: dendrite
 
-          - name: "Dendrite - :sqlite: / full HTTP APIs"
-            homeserver-implementation: dendrite
-            denrite-full-http: yes
+          - homeserver-implementation: dendrite
+            denrite-full-http: full-http
 
     container:
       image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}:${{ matrix.sytest-tag }}

From ed0d9d721f26692c6cada1c0cf5278e842916bc1 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 16:10:21 +0100
Subject: [PATCH 08/60] Maybe setting working dir will help?

---
 .github/workflows/pipeline.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index e48313b96..3392c90b4 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -62,6 +62,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Run sytest
+        working-directory: /sytest
         run: bash /bootstrap.sh ${{ matrix.homeserver-implementation }}
       - name: Summarise results.tap
         if: ${{ always() }}

From 31422957d097a7297eddb635ddaa477ce96efa99 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 16:22:12 +0100
Subject: [PATCH 09/60] Mount . as /sytest

---
 .github/workflows/pipeline.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 3392c90b4..4b7e8e325 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -53,6 +53,8 @@ jobs:
       image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}:${{ matrix.sytest-tag }}
       volumes:
         - ${{ github.workspace }}:/src
+        # TODO PWD mounted as /sytest was originally mounted as readonly. Could pass -v directly
+        - ".:/sytest"
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}

From f93a663c73afb136063c83e29a7421bc6259ad5f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 16:38:25 +0100
Subject: [PATCH 10/60] `.` was too short. Workspace as full path?

---
 .github/workflows/pipeline.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 4b7e8e325..87c0847d0 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -52,9 +52,9 @@ jobs:
     container:
       image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}:${{ matrix.sytest-tag }}
       volumes:
-        - ${{ github.workspace }}:/src
-        # TODO PWD mounted as /sytest was originally mounted as readonly. Could pass -v directly
-        - ".:/sytest"
+        # TODO PWD mounted as /sytest was originally mounted as readonly. Could pass -v directly?
+        - ${{ github.workspace }}:/sytest
+        - ./logs:/logs
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}

From 78814e084efca400e26f3475c35466b997e48f3c Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 16:43:24 +0100
Subject: [PATCH 11/60] Guess to try and fix the logs path

---
 .github/workflows/pipeline.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 87c0847d0..6acce6f87 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -50,11 +50,12 @@ jobs:
             denrite-full-http: full-http
 
     container:
-      image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}:${{ matrix.sytest-tag }}
+      # The colon is icky. Better way? Could just include the image explicitly?
+      image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}${{ matrix.sytest-tag && ":" }}${{ matrix.sytest-tag }}
       volumes:
         # TODO PWD mounted as /sytest was originally mounted as readonly. Could pass -v directly?
         - ${{ github.workspace }}:/sytest
-        - ./logs:/logs
+        - ${{ github.workspace }}/logs:/logs
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}

From 47ae955900bb54110da4f41b44ba98c4ffad1c18 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 16:45:14 +0100
Subject: [PATCH 12/60] single quotes :(

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 6acce6f87..c1187ce46 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -51,7 +51,7 @@ jobs:
 
     container:
       # The colon is icky. Better way? Could just include the image explicitly?
-      image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}${{ matrix.sytest-tag && ":" }}${{ matrix.sytest-tag }}
+      image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}${{ matrix.sytest-tag && ':' }}${{ matrix.sytest-tag }}
       volumes:
         # TODO PWD mounted as /sytest was originally mounted as readonly. Could pass -v directly?
         - ${{ github.workspace }}:/sytest

From f9ac10052b03e055302269c66525a2678f0afda9 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 16:51:49 +0100
Subject: [PATCH 13/60] bash -xe for debug info

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index c1187ce46..9d8522d3a 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -66,7 +66,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Run sytest
         working-directory: /sytest
-        run: bash /bootstrap.sh ${{ matrix.homeserver-implementation }}
+        run: bash -xe /bootstrap.sh ${{ matrix.homeserver-implementation }}
       - name: Summarise results.tap
         if: ${{ always() }}
         run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap

From d87a25e9bbeaa8615d361de610117a47f30ef072 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 17:03:53 +0100
Subject: [PATCH 14/60] more -xe

---
 scripts/synapse_sytest.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index 8db7c477a..98e11aee4 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -xe
 #
 # This script is run by the bootstrap.sh script in the docker image.
 #

From 1e98e8de9e58753a37012b6274e0359e2101380a Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 17:15:25 +0100
Subject: [PATCH 15/60] More exploratory debugging

---
 docker/bootstrap.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index b700474f4..015684bd4 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -7,16 +7,18 @@ set -ex
 export SYTEST_TARGET="$1"
 shift
 
+env | sort
+
 if [ -d "/sytest" ]; then
     # If the user has mounted in a SyTest checkout, use that.
     echo "Using local sytests"
 else
     echo "--- Trying to get same-named sytest branch..."
 
-    # Check if we're running in buildkite, if so it can tell us what
+    # Check if we're running under GitHub Actions. If so it can tell us what
     # Synapse/Dendrite branch we're running
-    if [ -n "$BUILDKITE_BRANCH" ]; then
-        branch_name=$BUILDKITE_BRANCH
+    if [ -n "$GITHUB_HEAD_REF" ]; then
+        branch_name=$GITHUB_HEAD_REF
     else
         # Otherwise, try and find the branch that the Synapse/Dendrite checkout
         # is using. Fall back to develop if unknown.

From 4a66f6d2ebcf308687a73e22278b6e9625e883e2 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 17:19:57 +0100
Subject: [PATCH 16/60] Try an explicit cat?

---
 docker/bootstrap.sh | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index 015684bd4..8d585b0d2 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -7,23 +7,17 @@ set -ex
 export SYTEST_TARGET="$1"
 shift
 
-env | sort
+env | sort | cat
 
 if [ -d "/sytest" ]; then
     # If the user has mounted in a SyTest checkout, use that.
+    # This is the case for sytest's GitHub Actions.
     echo "Using local sytests"
 else
     echo "--- Trying to get same-named sytest branch..."
-
-    # Check if we're running under GitHub Actions. If so it can tell us what
-    # Synapse/Dendrite branch we're running
-    if [ -n "$GITHUB_HEAD_REF" ]; then
-        branch_name=$GITHUB_HEAD_REF
-    else
-        # Otherwise, try and find the branch that the Synapse/Dendrite checkout
-        # is using. Fall back to develop if unknown.
-        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
-    fi
+    # Otherwise, try and find the branch that the Synapse/Dendrite checkout
+    # is using. Fall back to develop if unknown.
+    branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
 
     if [ "$SYTEST_TARGET" == "dendrite" ] && [ "$branch_name" == "master" ]; then
         # Dendrite uses master as its main branch. If the branch is master, we probably want sytest develop

From d87f6a8514ca9c3b210d3e21898240bdef573812 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 17:21:46 +0100
Subject: [PATCH 17/60] Comment out to make the pipeline simpler

---
 .github/workflows/pipeline.yml | 60 +++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 9d8522d3a..2558c8bba 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -19,35 +19,35 @@ jobs:
           - homeserver-implementation: synapse
             sytest-tag: bionic
 
-          - homeserver-implementation: synapse
-            sytest-tag: bionic
-            postgres: postgres
-
-          - homeserver-implementation: synapse
-            sytest-tag: bionic
-            postgres: postgres
-            synapse-workers: workers
-
-          - homeserver-implementation: synapse
-            sytest-tag: testing
-            postgres: postgres
-
-          - homeserver-implementation: synapse
-            sytest-tag: testing
-            postgres: postgres
-            synapse-workers: workers
-
-          - homeserver-implementation: dendrite
-            postgres: postgres
-
-          - homeserver-implementation: dendrite
-            postgres: postgres
-            denrite-full-http: full-http
-
-          - homeserver-implementation: dendrite
-
-          - homeserver-implementation: dendrite
-            denrite-full-http: full-http
+#          - homeserver-implementation: synapse
+#            sytest-tag: bionic
+#            postgres: postgres
+#
+#          - homeserver-implementation: synapse
+#            sytest-tag: bionic
+#            postgres: postgres
+#            synapse-workers: workers
+#
+#          - homeserver-implementation: synapse
+#            sytest-tag: testing
+#            postgres: postgres
+#
+#          - homeserver-implementation: synapse
+#            sytest-tag: testing
+#            postgres: postgres
+#            synapse-workers: workers
+#
+#          - homeserver-implementation: dendrite
+#            postgres: postgres
+#
+#          - homeserver-implementation: dendrite
+#            postgres: postgres
+#            denrite-full-http: full-http
+#
+#          - homeserver-implementation: dendrite
+#
+#          - homeserver-implementation: dendrite
+#            denrite-full-http: full-http
 
     container:
       # The colon is icky. Better way? Could just include the image explicitly?
@@ -68,7 +68,7 @@ jobs:
         working-directory: /sytest
         run: bash -xe /bootstrap.sh ${{ matrix.homeserver-implementation }}
       - name: Summarise results.tap
-        if: ${{ always() }}
+#        if: ${{ always() }}
         run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: TODO upload artifacts
         run: "false"

From 74362239633195dfc626ebd52e5941c1bc5b87ac Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 17:26:55 +0100
Subject: [PATCH 18/60] Where are we doing the tarring in?

---
 scripts/synapse_sytest.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index 98e11aee4..1da2d74bf 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -119,6 +119,9 @@ fi
 # it will otherwise try to build it in-tree, which means writing changes to the
 # source volume outside the container.)
 #
+
+echo Help I\'m stuck in $PWD
+
 if [ -d "$SYNAPSE_SOURCE" ]; then
     echo "Creating tarball from synapse source"
     tar -C "$SYNAPSE_SOURCE" -czf /tmp/synapse.tar.gz \

From 1e8f5ad2854357f8c02fd5777aef75ed32792b0a Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 17:50:42 +0100
Subject: [PATCH 19/60] synapse_sytest: use SYNAPSE_SOURCE throughout

---
 scripts/synapse_sytest.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index 1da2d74bf..f77ce729e 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -2,9 +2,9 @@
 #
 # This script is run by the bootstrap.sh script in the docker image.
 #
-# It expects to find the synapse source in /src, and a virtualenv in /venv.
-# It installs synapse into the virtualenv, configures sytest according to the
-# env vars, and runs sytest.
+# It expects to find the synapse source in $SYNAPSE_SOURCE (default /src),
+# and a virtualenv in /venv. It installs synapse into the virtualenv,
+# configures sytest according to the env vars, and runs sytest.
 #
 
 # Run the sytests.
@@ -160,10 +160,10 @@ fi
 # Run the tests
 echo >&2 "+++ Running tests"
 
-export COVERAGE_PROCESS_START="/src/.coveragerc"
+export COVERAGE_PROCESS_START="$SYNAPSE_SOURCE/.coveragerc"
 
 RUN_TESTS=(
-    perl -I "$SYTEST_LIB" /sytest/run-tests.pl --python=/venv/bin/python --synapse-directory=/src -B "/src/$BLACKLIST" --coverage -O tap --all
+    perl -I "$SYTEST_LIB" /sytest/run-tests.pl --python=/venv/bin/python --synapse-directory="$SYNAPSE_SOURCE" -B "$SYNAPSE_SOURCe/$BLACKLIST" --coverage -O tap --all
     --work-directory="/work"
 )
 
@@ -198,10 +198,10 @@ echo >&2 "--- Copying assets"
 
 # Copy out the logs
 rsync --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
-#cp /.coverage.* /src || true
+#cp /.coverage.* "$SYNAPSE_SOURCE" || true
 
-#cd /src
-#export TOP=/src
+#cd "$SYNAPSE_SOURCE"
+#export TOP="$SYNAPSE_SOURCE"
 #/venv/bin/coverage combine
 
 if [ $TEST_STATUS -ne 0 ]; then

From 04329aeab6244a99356d8944bdd091edfa7879fa Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 17:51:02 +0100
Subject: [PATCH 20/60] Can we get sytest running?

---
 .github/workflows/pipeline.yml | 6 +++++-
 docker/bootstrap.sh            | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 2558c8bba..d89a364f3 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -53,8 +53,12 @@ jobs:
       # The colon is icky. Better way? Could just include the image explicitly?
       image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}${{ matrix.sytest-tag && ':' }}${{ matrix.sytest-tag }}
       volumes:
-        # TODO PWD mounted as /sytest was originally mounted as readonly. Could pass -v directly?
+        # Bootstrap script expects the sytest source available at /sytest.
+        # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
         - ${{ github.workspace }}:/sytest
+        # synapse_sytest.sh seems to expect it at /src, but the bootstrap script will feed
+        # it /sytest via an environment variable
+        # - ${{ github.workspace }}:/src
         - ${{ github.workspace }}/logs:/logs
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index 8d585b0d2..5b66b0bef 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -53,6 +53,7 @@ echo "--- Preparing sytest for ${SYTEST_TARGET}"
 export SYTEST_LIB="/sytest/lib"
 
 if [ -x "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" ]; then
+    export SYNAPSE_SOURCE="/sytest"
     exec "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" "$@"
 
 elif [ -x "/sytest/docker/${SYTEST_TARGET}_sytest.sh" ]; then

From 06f684baf052bfa141afcb15cd8238455488740e Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 18:00:14 +0100
Subject: [PATCH 21/60] stupid debug

---
 docker/bootstrap.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index 5b66b0bef..d460045a0 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -53,6 +53,7 @@ echo "--- Preparing sytest for ${SYTEST_TARGET}"
 export SYTEST_LIB="/sytest/lib"
 
 if [ -x "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" ]; then
+    echo hello I am really running this line
     export SYNAPSE_SOURCE="/sytest"
     exec "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" "$@"
 

From bc453bab6a5e24628bcfc11eeb220ea3a7e0102b Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 18:04:10 +0100
Subject: [PATCH 22/60] desperation printf debugging

---
 docker/bootstrap.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index d460045a0..a7978c54a 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -52,16 +52,20 @@ echo "--- Preparing sytest for ${SYTEST_TARGET}"
 
 export SYTEST_LIB="/sytest/lib"
 
+
+echo DMR 1
 if [ -x "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" ]; then
-    echo hello I am really running this line
+    echo DMR 2
     export SYNAPSE_SOURCE="/sytest"
     exec "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" "$@"
 
 elif [ -x "/sytest/docker/${SYTEST_TARGET}_sytest.sh" ]; then
+    echo DMR 3
     # old branches of sytest used to put the sytest running script in the "/docker" directory
     exec "/sytest/docker/${SYTEST_TARGET}_sytest.sh" "$@"
 
 else
+    echo DMR 4
     PLUGIN_RUNNER=$(find /sytest/plugins/ -type f -name "${SYTEST_TARGET}_sytest.sh" -print)
     if [ -n PLUGIN_RUNNER ]; then
         exec ${PLUGIN_RUNNER} "$@"

From 0567858ca16632a4e3e0037de7a27db7a80ece85 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 18:08:18 +0100
Subject: [PATCH 23/60] echo with quotes!?!?!

---
 docker/bootstrap.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index a7978c54a..7b76f650a 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -53,19 +53,19 @@ echo "--- Preparing sytest for ${SYTEST_TARGET}"
 export SYTEST_LIB="/sytest/lib"
 
 
-echo DMR 1
+echo "DMR 1"
 if [ -x "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" ]; then
-    echo DMR 2
+    echo "DMR 2"
     export SYNAPSE_SOURCE="/sytest"
     exec "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" "$@"
 
 elif [ -x "/sytest/docker/${SYTEST_TARGET}_sytest.sh" ]; then
-    echo DMR 3
+    echo "DMR 3"
     # old branches of sytest used to put the sytest running script in the "/docker" directory
     exec "/sytest/docker/${SYTEST_TARGET}_sytest.sh" "$@"
 
 else
-    echo DMR 4
+    echo "DMR 4"
     PLUGIN_RUNNER=$(find /sytest/plugins/ -type f -name "${SYTEST_TARGET}_sytest.sh" -print)
     if [ -n PLUGIN_RUNNER ]; then
         exec ${PLUGIN_RUNNER} "$@"

From 11711299289a94a1c145b398a3f93d7d8316c208 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 18:29:02 +0100
Subject: [PATCH 24/60] Remove bootstrap debug for now; mount /src

bootstrap script is in the sytest-* containers, and I was running their
version instead of the version in the checkout. Confusing!!
---
 .github/workflows/pipeline.yml | 6 +++---
 docker/bootstrap.sh            | 8 --------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index d89a364f3..58f3d6a92 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -56,9 +56,9 @@ jobs:
         # Bootstrap script expects the sytest source available at /sytest.
         # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
         - ${{ github.workspace }}:/sytest
-        # synapse_sytest.sh seems to expect it at /src, but the bootstrap script will feed
-        # it /sytest via an environment variable
-        # - ${{ github.workspace }}:/src
+        # But synapse_sytest.sh expects it at /src
+        # TODO could change the bootstrap script to pass SYTEST_SOURCE=/sytest
+        - ${{ github.workspace }}:/src
         - ${{ github.workspace }}/logs:/logs
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index 7b76f650a..e0ec9e4f5 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -7,8 +7,6 @@ set -ex
 export SYTEST_TARGET="$1"
 shift
 
-env | sort | cat
-
 if [ -d "/sytest" ]; then
     # If the user has mounted in a SyTest checkout, use that.
     # This is the case for sytest's GitHub Actions.
@@ -52,20 +50,14 @@ echo "--- Preparing sytest for ${SYTEST_TARGET}"
 
 export SYTEST_LIB="/sytest/lib"
 
-
-echo "DMR 1"
 if [ -x "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" ]; then
-    echo "DMR 2"
-    export SYNAPSE_SOURCE="/sytest"
     exec "/sytest/scripts/${SYTEST_TARGET}_sytest.sh" "$@"
 
 elif [ -x "/sytest/docker/${SYTEST_TARGET}_sytest.sh" ]; then
-    echo "DMR 3"
     # old branches of sytest used to put the sytest running script in the "/docker" directory
     exec "/sytest/docker/${SYTEST_TARGET}_sytest.sh" "$@"
 
 else
-    echo "DMR 4"
     PLUGIN_RUNNER=$(find /sytest/plugins/ -type f -name "${SYTEST_TARGET}_sytest.sh" -print)
     if [ -n PLUGIN_RUNNER ]; then
         exec ${PLUGIN_RUNNER} "$@"

From b8a7175fcb77d99cd86520d53fb889c988b008a9 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 19:08:12 +0100
Subject: [PATCH 25/60] I think it'll be easier to concentrate on synapse

---
 .github/workflows/pipeline.yml | 81 ++++++++++++++++++----------------
 1 file changed, 42 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 58f3d6a92..595160cf8 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -16,58 +16,49 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - homeserver-implementation: synapse
-            sytest-tag: bionic
+          - sytest-tag: bionic
 
-#          - homeserver-implementation: synapse
-#            sytest-tag: bionic
-#            postgres: postgres
-#
-#          - homeserver-implementation: synapse
-#            sytest-tag: bionic
-#            postgres: postgres
-#            synapse-workers: workers
-#
-#          - homeserver-implementation: synapse
-#            sytest-tag: testing
-#            postgres: postgres
-#
-#          - homeserver-implementation: synapse
-#            sytest-tag: testing
-#            postgres: postgres
-#            synapse-workers: workers
-#
-#          - homeserver-implementation: dendrite
-#            postgres: postgres
-#
-#          - homeserver-implementation: dendrite
-#            postgres: postgres
-#            denrite-full-http: full-http
-#
-#          - homeserver-implementation: dendrite
-#
-#          - homeserver-implementation: dendrite
-#            denrite-full-http: full-http
+          - sytest-tag: bionic
+            postgres: postgres
+
+          - sytest-tag: bionic
+            postgres: postgres
+            workers: workers
+
+          - sytest-tag: testing
+            postgres: postgres
+
+          - sytest-tag: testing
+            postgres: postgres
+            workers: workers
 
     container:
-      # The colon is icky. Better way? Could just include the image explicitly?
-      image: matrixdotorg/sytest-${{ matrix.homeserver-implementation }}${{ matrix.sytest-tag && ':' }}${{ matrix.sytest-tag }}
+      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
       volumes:
-        # Bootstrap script expects the sytest source available at /sytest.
+        # bootstrap.sh expects the sytest source available at /sytest.
         # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
         - ${{ github.workspace }}:/sytest
-        # But synapse_sytest.sh expects it at /src
-        # TODO could change the bootstrap script to pass SYTEST_SOURCE=/sytest
-        - ${{ github.workspace }}:/src
-        - ${{ github.workspace }}/logs:/logs
+        # synapse_sytest.sh expects a synapse checkout at /src
+        - ${{ github.workspace }}/synapse:/src
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}
-        BLACKLIST: ${{ (matrix.synapse-workers && 'synapse-blacklist-with-workers') || 'synapse-blacklist' }}
+        BLACKLIST: ${{ (matrix.workers && 'synapse-blacklist-with-workers') || 'synapse-blacklist' }}
         API: ${{ matrix.dendrite-full-http && 1 }}
 
     steps:
       - uses: actions/checkout@v2
+      # From https://stackoverflow.com/a/58035262 . Surprising that GHA doesn't provide this by default?
+      - name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      - name: Checkout synapse
+        uses: actions/checkout@v2
+        with:
+          repository: matrix-org/synapse
+          ref: develop
+          path: 'src'
       - name: Run sytest
         working-directory: /sytest
         run: bash -xe /bootstrap.sh ${{ matrix.homeserver-implementation }}
@@ -78,3 +69,15 @@ jobs:
         run: "false"
       - name: TODO annotate logs
         run: "false"
+
+#          - homeserver-implementation: dendrite
+#            postgres: postgres
+#
+#          - homeserver-implementation: dendrite
+#            postgres: postgres
+#            denrite-full-http: full-http
+#
+#          - homeserver-implementation: dendrite
+#
+#          - homeserver-implementation: dendrite
+#            denrite-full-http: full-http

From 8ff4b2d9450b45d5a369f45f6460d523b54e505f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 19:10:24 +0100
Subject: [PATCH 26/60] Again

---
 .github/workflows/pipeline.yml | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 595160cf8..dab3db835 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -18,19 +18,19 @@ jobs:
         include:
           - sytest-tag: bionic
 
-          - sytest-tag: bionic
-            postgres: postgres
-
-          - sytest-tag: bionic
-            postgres: postgres
-            workers: workers
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-            workers: workers
+#          - sytest-tag: bionic
+#            postgres: postgres
+#
+#          - sytest-tag: bionic
+#            postgres: postgres
+#            workers: workers
+#
+#          - sytest-tag: testing
+#            postgres: postgres
+#
+#          - sytest-tag: testing
+#            postgres: postgres
+#            workers: workers
 
     container:
       image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
@@ -61,7 +61,7 @@ jobs:
           path: 'src'
       - name: Run sytest
         working-directory: /sytest
-        run: bash -xe /bootstrap.sh ${{ matrix.homeserver-implementation }}
+        run: bash -xe /bootstrap.sh synapse
       - name: Summarise results.tap
 #        if: ${{ always() }}
         run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap

From 5ed4665f6dbcd454b58d009a9d311ea558e616bb Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 19:22:26 +0100
Subject: [PATCH 27/60] Revert a bunch of incorrect changes; mkdir -p

Getting somewhere
---
 scripts/synapse_sytest.sh | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index f77ce729e..102a6df1e 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -2,9 +2,9 @@
 #
 # This script is run by the bootstrap.sh script in the docker image.
 #
-# It expects to find the synapse source in $SYNAPSE_SOURCE (default /src),
-# and a virtualenv in /venv. It installs synapse into the virtualenv,
-# configures sytest according to the env vars, and runs sytest.
+# It expects to find the synapse source in /src, and a virtualenv in /venv.
+# It installs synapse into the virtualenv, configures sytest according to the
+# env vars, and runs sytest.
 #
 
 # Run the sytests.
@@ -13,7 +13,7 @@ set -e
 
 cd "$(dirname $0)/.."
 
-mkdir /work
+mkdir -p /work
 
 # start the redis server, if desired
 if [ -n "$REDIS" ]; then
@@ -119,9 +119,6 @@ fi
 # it will otherwise try to build it in-tree, which means writing changes to the
 # source volume outside the container.)
 #
-
-echo Help I\'m stuck in $PWD
-
 if [ -d "$SYNAPSE_SOURCE" ]; then
     echo "Creating tarball from synapse source"
     tar -C "$SYNAPSE_SOURCE" -czf /tmp/synapse.tar.gz \
@@ -160,10 +157,10 @@ fi
 # Run the tests
 echo >&2 "+++ Running tests"
 
-export COVERAGE_PROCESS_START="$SYNAPSE_SOURCE/.coveragerc"
+export COVERAGE_PROCESS_START="/src/.coveragerc"
 
 RUN_TESTS=(
-    perl -I "$SYTEST_LIB" /sytest/run-tests.pl --python=/venv/bin/python --synapse-directory="$SYNAPSE_SOURCE" -B "$SYNAPSE_SOURCe/$BLACKLIST" --coverage -O tap --all
+    perl -I "$SYTEST_LIB" /sytest/run-tests.pl --python=/venv/bin/python --synapse-directory=/src -B "/src/$BLACKLIST" --coverage -O tap --all
     --work-directory="/work"
 )
 
@@ -198,10 +195,10 @@ echo >&2 "--- Copying assets"
 
 # Copy out the logs
 rsync --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
-#cp /.coverage.* "$SYNAPSE_SOURCE" || true
+#cp /.coverage.* /src || true
 
-#cd "$SYNAPSE_SOURCE"
-#export TOP="$SYNAPSE_SOURCE"
+#cd /src
+#export TOP=/src
 #/venv/bin/coverage combine
 
 if [ $TEST_STATUS -ne 0 ]; then

From 230247978487b4ca4168b61e253822f9d5dd91e5 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 19:26:05 +0100
Subject: [PATCH 28/60] Tweak pipeline

---
 .github/workflows/pipeline.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index dab3db835..6cbdf88c8 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -48,19 +48,19 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      # From https://stackoverflow.com/a/58035262 . Surprising that GHA doesn't provide this by default?
-      - name: Extract branch name
-        shell: bash
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: extract_branch
+#      # From https://stackoverflow.com/a/58035262 . Surprising that GHA doesn't provide this by default?
+#      - name: Extract branch name
+#        shell: bash
+#        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+#        id: extract_branch
       - name: Checkout synapse
         uses: actions/checkout@v2
         with:
           repository: matrix-org/synapse
+          # TODO get this to use our branch, if that exists in synapse
           ref: develop
-          path: 'src'
+          path: '/src'
       - name: Run sytest
-        working-directory: /sytest
         run: bash -xe /bootstrap.sh synapse
       - name: Summarise results.tap
 #        if: ${{ always() }}

From 5aab2fe3277c0375bf3dfb8cf981f4ef8fa1018b Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 19:31:25 +0100
Subject: [PATCH 29/60] Tweak checkout

---
 .github/workflows/pipeline.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 6cbdf88c8..3c804bbf7 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -37,7 +37,7 @@ jobs:
       volumes:
         # bootstrap.sh expects the sytest source available at /sytest.
         # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
-        - ${{ github.workspace }}:/sytest
+        - ${{ github.workspace }}/sytest:/sytest
         # synapse_sytest.sh expects a synapse checkout at /src
         - ${{ github.workspace }}/synapse:/src
       env:
@@ -47,7 +47,8 @@ jobs:
         API: ${{ matrix.dendrite-full-http && 1 }}
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout sytest
+        uses: actions/checkout@v2
 #      # From https://stackoverflow.com/a/58035262 . Surprising that GHA doesn't provide this by default?
 #      - name: Extract branch name
 #        shell: bash
@@ -59,7 +60,6 @@ jobs:
           repository: matrix-org/synapse
           # TODO get this to use our branch, if that exists in synapse
           ref: develop
-          path: '/src'
       - name: Run sytest
         run: bash -xe /bootstrap.sh synapse
       - name: Summarise results.tap

From ec135d6b2389b22bc58ed26a2136314052d2d1d3 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 19:33:45 +0100
Subject: [PATCH 30/60] Explicit paths

---
 .github/workflows/pipeline.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 3c804bbf7..bdd4da9ef 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -49,6 +49,8 @@ jobs:
     steps:
       - name: Checkout sytest
         uses: actions/checkout@v2
+        with:
+          path: sytest
 #      # From https://stackoverflow.com/a/58035262 . Surprising that GHA doesn't provide this by default?
 #      - name: Extract branch name
 #        shell: bash
@@ -58,6 +60,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: matrix-org/synapse
+          path: synapse
           # TODO get this to use our branch, if that exists in synapse
           ref: develop
       - name: Run sytest

From 2f0c6d20587ec2f9c7cba86259504b0153c6482f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 19:40:30 +0100
Subject: [PATCH 31/60] Fix blacklist path

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index bdd4da9ef..21ceb0f53 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -43,7 +43,7 @@ jobs:
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}
-        BLACKLIST: ${{ (matrix.workers && 'synapse-blacklist-with-workers') || 'synapse-blacklist' }}
+        BLACKLIST: ${{ (matrix.workers && 'ci/worker-blacklist') || 'sytest-blacklist' }}
         API: ${{ matrix.dendrite-full-http && 1 }}
 
     steps:

From cfb8729ecfcc60ff9ef136c27692a454d9122587 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 20:23:07 +0100
Subject: [PATCH 32/60] Try to get sytest logs uploaded as artefacts

---
 .github/workflows/pipeline.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 21ceb0f53..7a13307df 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -68,8 +68,14 @@ jobs:
       - name: Summarise results.tap
 #        if: ${{ always() }}
         run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: TODO upload artifacts
-        run: "false"
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (Synapse, ${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
       - name: TODO annotate logs
         run: "false"
 

From b78fad20dd1e2a8df3658fb682945e60c2cfd6da Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 11 Aug 2021 20:23:27 +0100
Subject: [PATCH 33/60] update TODOs; try more synapse testing

---
 .github/workflows/pipeline.yml | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 7a13307df..a32a521c6 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -18,19 +18,19 @@ jobs:
         include:
           - sytest-tag: bionic
 
-#          - sytest-tag: bionic
-#            postgres: postgres
-#
-#          - sytest-tag: bionic
-#            postgres: postgres
-#            workers: workers
-#
-#          - sytest-tag: testing
-#            postgres: postgres
-#
-#          - sytest-tag: testing
-#            postgres: postgres
-#            workers: workers
+          - sytest-tag: bionic
+            postgres: postgres
+
+          - sytest-tag: bionic
+            postgres: postgres
+            workers: workers
+
+          - sytest-tag: testing
+            postgres: postgres
+
+          - sytest-tag: testing
+            postgres: postgres
+            workers: workers
 
     container:
       image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
@@ -56,12 +56,14 @@ jobs:
 #        shell: bash
 #        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
 #        id: extract_branch
+      # TODO get this to use our branch, if that exists in synapse
+      # TODO synapse has a similar need when its tests.yml checks out complement.
+      # Could we turn "Checkout complement" into a new GitHub Action?
       - name: Checkout synapse
         uses: actions/checkout@v2
         with:
           repository: matrix-org/synapse
           path: synapse
-          # TODO get this to use our branch, if that exists in synapse
           ref: develop
       - name: Run sytest
         run: bash -xe /bootstrap.sh synapse

From 0862c20d9cf747581f3f9dd8b09af3970f8e8cfa Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 11:17:55 +0100
Subject: [PATCH 34/60] Try to checkout the corresponding synapse branch

---
 .github/workflows/pipeline.yml | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index a32a521c6..a6748b4f9 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -51,25 +51,37 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: sytest
-#      # From https://stackoverflow.com/a/58035262 . Surprising that GHA doesn't provide this by default?
-#      - name: Extract branch name
-#        shell: bash
-#        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-#        id: extract_branch
-      # TODO get this to use our branch, if that exists in synapse
-      # TODO synapse has a similar need when its tests.yml checks out complement.
-      # Could we turn "Checkout complement" into a new GitHub Action?
-      - name: Checkout synapse
+
+      # From https://stackoverflow.com/a/58035262 . Surprising that GHA doesn't provide this by default?
+      - name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+
+      - name: Try to checkout synapse `${{ steps.extract_branch.outputs.branch }}` branch
+        id: checkout_synapse_branch
+        uses: actions/checkout@v2
+        with:
+          repository: matrix-org/synapse
+          path: synapse
+          ref: ${{ steps.extract_branch.outputs.branch }}
+        continue-on-error: true
+
+      - name: Checkout synapse `develop` branch
+        if: ${{ steps.checkout_synapse_branch.outcome != 'success' }}
         uses: actions/checkout@v2
         with:
           repository: matrix-org/synapse
           path: synapse
           ref: develop
+
       - name: Run sytest
         run: bash -xe /bootstrap.sh synapse
+
       - name: Summarise results.tap
-#        if: ${{ always() }}
+        if: ${{ always() }}
         run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
+
       - name: Upload SyTest logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
@@ -78,6 +90,7 @@ jobs:
           path: |
             /logs/results.tap
             /logs/**/*.log*
+
       - name: TODO annotate logs
         run: "false"
 

From 45cbb1aa9a801bf7b2c9457e67564bf8e0ab4648 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 11:47:09 +0100
Subject: [PATCH 35/60] Use rest API to avoid checkouts action retrying

---
 .github/workflows/pipeline.yml | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index a6748b4f9..043f20e51 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -52,28 +52,37 @@ jobs:
         with:
           path: sytest
 
-      # From https://stackoverflow.com/a/58035262 . Surprising that GHA doesn't provide this by default?
+      # From https://stackoverflow.com/a/58035262
       - name: Extract branch name
         shell: bash
         run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
         id: extract_branch
 
-      - name: Try to checkout synapse `${{ steps.extract_branch.outputs.branch }}` branch
-        id: checkout_synapse_branch
-        uses: actions/checkout@v2
+      - name: Find synapse `${{ steps.extract_branch.outputs.branch }}` branch
+        id: find_synapse_branch
+        uses: octokit/request-action@v2.1.0
         with:
-          repository: matrix-org/synapse
-          path: synapse
+          route: GET /repos/matrix-org/synapse/branches/${{ steps.extract_branch.outputs.branch }}
           ref: ${{ steps.extract_branch.outputs.branch }}
         continue-on-error: true
 
-      - name: Checkout synapse `develop` branch
-        if: ${{ steps.checkout_synapse_branch.outcome != 'success' }}
+      - name: Select synapse branch to checkout
+        id: select_synapse_branch
+        shell: bash
+        run: |
+          if [ ${{ steps.find_synapse_branch.outcome }} == "success" ]; then
+            branch=${{ steps.extract_branch.outputs.branch }}
+          else
+            branch=develop
+          fi
+          echo "::set-output name=branch::$branch"
+
+      - name: Checkout synapse `${{ steps.select_synapse_branch.outputs.branch }}` branch
         uses: actions/checkout@v2
         with:
           repository: matrix-org/synapse
           path: synapse
-          ref: develop
+          ref: ${{ steps.select_synapse_branch.outputs.branch }}
 
       - name: Run sytest
         run: bash -xe /bootstrap.sh synapse

From ccd4e8981cb609be23c67ed7cfe4066e24faca61 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 11:52:49 +0100
Subject: [PATCH 36/60] Need API token to make an API request hurrrr

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 043f20e51..86f2a08e3 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -62,8 +62,8 @@ jobs:
         id: find_synapse_branch
         uses: octokit/request-action@v2.1.0
         with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           route: GET /repos/matrix-org/synapse/branches/${{ steps.extract_branch.outputs.branch }}
-          ref: ${{ steps.extract_branch.outputs.branch }}
         continue-on-error: true
 
       - name: Select synapse branch to checkout

From 7eafc11d54ce3bcaf04225fcc03cce7f5beae27d Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 13:16:06 +0100
Subject: [PATCH 37/60] Log annotation already handled by tap_to_gha.pl

---
 .github/workflows/pipeline.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 86f2a08e3..b0d618dc7 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -100,9 +100,6 @@ jobs:
             /logs/results.tap
             /logs/**/*.log*
 
-      - name: TODO annotate logs
-        run: "false"
-
 #          - homeserver-implementation: dendrite
 #            postgres: postgres
 #

From 66f1ac5d9315d56ff1c3489132cd97a503ef07df Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 13:16:34 +0100
Subject: [PATCH 38/60] ci -> .ci

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index b0d618dc7..83cc24769 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -43,7 +43,7 @@ jobs:
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}
-        BLACKLIST: ${{ (matrix.workers && 'ci/worker-blacklist') || 'sytest-blacklist' }}
+        BLACKLIST: ${{ (matrix.workers && '.ci/worker-blacklist') || 'sytest-blacklist' }}
         API: ${{ matrix.dendrite-full-http && 1 }}
 
     steps:

From d2e27df4307266fc1e93d3b7a90590c3f4ecf080 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 13:16:47 +0100
Subject: [PATCH 39/60] Remove backticks

that don't work son
---
 .github/workflows/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 83cc24769..b65ed79f6 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -58,7 +58,7 @@ jobs:
         run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
         id: extract_branch
 
-      - name: Find synapse `${{ steps.extract_branch.outputs.branch }}` branch
+      - name: Find synapse ${{ steps.extract_branch.outputs.branch }} branch
         id: find_synapse_branch
         uses: octokit/request-action@v2.1.0
         with:
@@ -77,7 +77,7 @@ jobs:
           fi
           echo "::set-output name=branch::$branch"
 
-      - name: Checkout synapse `${{ steps.select_synapse_branch.outputs.branch }}` branch
+      - name: Checkout synapse ${{ steps.select_synapse_branch.outputs.branch }} branch
         uses: actions/checkout@v2
         with:
           repository: matrix-org/synapse

From 58aabb988021c61da27bad5cc339c192db542934 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 13:17:00 +0100
Subject: [PATCH 40/60] dendrite

---
 .github/workflows/pipeline.yml | 85 +++++++++++++++++++++++++++++-----
 1 file changed, 74 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index b65ed79f6..8f2fa22c8 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -100,14 +100,77 @@ jobs:
             /logs/results.tap
             /logs/**/*.log*
 
-#          - homeserver-implementation: dendrite
-#            postgres: postgres
-#
-#          - homeserver-implementation: dendrite
-#            postgres: postgres
-#            denrite-full-http: full-http
-#
-#          - homeserver-implementation: dendrite
-#
-#          - homeserver-implementation: dendrite
-#            denrite-full-http: full-http
+  dendrite:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        db: ["sqlite", "postgres"]
+        api: ["not-full-http", "full-http"]
+
+    container:
+      image: matrixdotorg/sytest-dendrite
+      volumes:
+        # bootstrap.sh expects the sytest source available at /sytest.
+        # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
+        - ${{ github.workspace }}/sytest:/sytest
+        # synapse_sytest.sh expects a synapse checkout at /src
+        - ${{ github.workspace }}/synapse:/src
+      env:
+        POSTGRES: ${{ matrix.db == 'postgres' }}
+        API: ${{ matrix.api == 'full-http' }}
+
+    steps:
+      - name: Checkout sytest
+        uses: actions/checkout@v2
+        with:
+          path: sytest
+
+      # From https://stackoverflow.com/a/58035262
+      - name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+
+      - name: Find dendrite ${{ steps.extract_branch.outputs.branch }} branch
+        id: find_dendrite_branch
+        uses: octokit/request-action@v2.1.0
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          route: GET /repos/matrix-org/dendrite/branches/${{ steps.extract_branch.outputs.branch }}
+        continue-on-error: true
+
+      - name: Select dendrite branch to checkout
+        id: select_dendrite_branch
+        shell: bash
+        run: |
+          if [ ${{ steps.find_dendrite_branch.outcome }} == "success" ]; then
+            branch=${{ steps.extract_branch.outputs.branch }}
+          else
+            branch=master
+          fi
+          echo "::set-output name=branch::$branch"
+
+      - name: Checkout dendrite ${{ steps.select_dendrite_branch.outputs.branch }} branch
+        uses: actions/checkout@v2
+        with:
+          repository: matrix-org/dendrite
+          path: dendrite
+          ref: ${{ steps.select_dendrite_branch.outputs.branch }}
+
+      - name: Run sytest
+        run: bash -xe /bootstrap.sh dendrite
+
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
+
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*

From c74afbdadb6b3cedf33ee444ed2b9ae9eb3671c0 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 13:22:14 +0100
Subject: [PATCH 41/60] Fix docker volume cfg to mount dendrite

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 8f2fa22c8..cfafa70d3 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -39,7 +39,7 @@ jobs:
         # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
         - ${{ github.workspace }}/sytest:/sytest
         # synapse_sytest.sh expects a synapse checkout at /src
-        - ${{ github.workspace }}/synapse:/src
+        - ${{ github.workspace }}/dendrite:/src
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}

From 13f9000b0468aa941990ef60b14816fb521bdeac Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 13:29:56 +0100
Subject: [PATCH 42/60] I got lost in the yaml

---
 .github/workflows/pipeline.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index cfafa70d3..61d082fc3 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -9,7 +9,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  sytest:
+  synapse:
     runs-on: ubuntu-latest
 
     strategy:
@@ -39,7 +39,7 @@ jobs:
         # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
         - ${{ github.workspace }}/sytest:/sytest
         # synapse_sytest.sh expects a synapse checkout at /src
-        - ${{ github.workspace }}/dendrite:/src
+        - ${{ github.workspace }}/synapse:/src
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}
@@ -116,7 +116,7 @@ jobs:
         # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
         - ${{ github.workspace }}/sytest:/sytest
         # synapse_sytest.sh expects a synapse checkout at /src
-        - ${{ github.workspace }}/synapse:/src
+        - ${{ github.workspace }}/dendrite:/src
       env:
         POSTGRES: ${{ matrix.db == 'postgres' }}
         API: ${{ matrix.api == 'full-http' }}

From bc2b9ced31455a1e912236a017ee22c2540d1e25 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 14:18:52 +0100
Subject: [PATCH 43/60] Try to reduce annotation spam

---
 .github/workflows/pipeline.yml | 35 +++++++++++++++-------------------
 scripts/dendrite_sytest.sh     |  2 +-
 scripts/synapse_sytest.sh      |  2 +-
 3 files changed, 17 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 61d082fc3..0699201b3 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -52,30 +52,24 @@ jobs:
         with:
           path: sytest
 
-      # From https://stackoverflow.com/a/58035262
-      - name: Extract branch name
-        shell: bash
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: extract_branch
-
-      - name: Find synapse ${{ steps.extract_branch.outputs.branch }} branch
-        id: find_synapse_branch
-        uses: octokit/request-action@v2.1.0
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          route: GET /repos/matrix-org/synapse/branches/${{ steps.extract_branch.outputs.branch }}
-        continue-on-error: true
-
-      - name: Select synapse branch to checkout
+      - name: Select corresponding synapse branch
         id: select_synapse_branch
         shell: bash
         run: |
-          if [ ${{ steps.find_synapse_branch.outcome }} == "success" ]; then
-            branch=${{ steps.extract_branch.outputs.branch }}
+          BRANCH=${GITHUB_REF#refs/heads/}
+          STATUS_CODE=$(curl -X GET \
+            -H "Accept: application/vnd.github.v3+json" \
+            -H "Authorization: token $GITHUB_TOKEN" \
+            -o /dev/null -w "%{http_code}"
+            https://api.github.com/repos/matrix-org/synapse/branches/$BRANCH
+          )
+          if [ STATUS_CODE == 200 ]; then
+            echo ::set-output name=branch:$BRANCH
           else
-            branch=develop
+            echo ::set-output name=branch:develop
           fi
-          echo "::set-output name=branch::$branch"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Checkout synapse ${{ steps.select_synapse_branch.outputs.branch }} branch
         uses: actions/checkout@v2
@@ -137,8 +131,9 @@ jobs:
         id: find_dendrite_branch
         uses: octokit/request-action@v2.1.0
         with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           route: GET /repos/matrix-org/dendrite/branches/${{ steps.extract_branch.outputs.branch }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         continue-on-error: true
 
       - name: Select dendrite branch to checkout
diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh
index cbe674033..08d68fc76 100755
--- a/scripts/dendrite_sytest.sh
+++ b/scripts/dendrite_sytest.sh
@@ -70,7 +70,7 @@ rsync -r --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /
 
 if [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation
-    perl /sytest/scripts/format_tap.pl /logs/results.tap "$BUILDKITE_LABEL" >/logs/annotate.md
+    perl /sytest/scripts/format_tap.pl /logs/results.tap "$GITHUB_SHA" >/logs/annotate.md
     # If show-expected-fail-tests logged something, put it into the annotation
     # Annotations from a failed build show at the top of buildkite, alerting
     # developers quickly as to what needs to change in the black/whitelist.
diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index 102a6df1e..7f31dbbfe 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -203,7 +203,7 @@ rsync --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /log
 
 if [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation
-    perl /sytest/scripts/format_tap.pl /logs/results.tap "$BUILDKITE_LABEL" >/logs/annotate.md
+    perl /sytest/scripts/format_tap.pl /logs/results.tap "$GITHUB_SHA" >/logs/annotate.md
 fi
 
 exit $TEST_STATUS

From aec7fa6ca6fff5cb6032d553b7855ab271c0db0f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 14:23:48 +0100
Subject: [PATCH 44/60] Okay, that didn't work because no curl

---
 .github/workflows/pipeline.yml | 24 ++----------------------
 1 file changed, 2 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 0699201b3..b8a66d13c 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -52,31 +52,11 @@ jobs:
         with:
           path: sytest
 
-      - name: Select corresponding synapse branch
-        id: select_synapse_branch
+      - name: Fetch corresponding synapse branch
         shell: bash
         run: |
           BRANCH=${GITHUB_REF#refs/heads/}
-          STATUS_CODE=$(curl -X GET \
-            -H "Accept: application/vnd.github.v3+json" \
-            -H "Authorization: token $GITHUB_TOKEN" \
-            -o /dev/null -w "%{http_code}"
-            https://api.github.com/repos/matrix-org/synapse/branches/$BRANCH
-          )
-          if [ STATUS_CODE == 200 ]; then
-            echo ::set-output name=branch:$BRANCH
-          else
-            echo ::set-output name=branch:develop
-          fi
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Checkout synapse ${{ steps.select_synapse_branch.outputs.branch }} branch
-        uses: actions/checkout@v2
-        with:
-          repository: matrix-org/synapse
-          path: synapse
-          ref: ${{ steps.select_synapse_branch.outputs.branch }}
+          (wget -O - https://github.com/matrix-org/synapse/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/synapse/archive/master.tar.gz) | tar -xz --strip-components=1 -C /src/"
 
       - name: Run sytest
         run: bash -xe /bootstrap.sh synapse

From 1a6a46c01aedc9bb81c96de36f784ca954422557 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 14:26:18 +0100
Subject: [PATCH 45/60] Missing quote

---
 .github/workflows/pipeline.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index b8a66d13c..22981a403 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -56,7 +56,8 @@ jobs:
         shell: bash
         run: |
           BRANCH=${GITHUB_REF#refs/heads/}
-          (wget -O - https://github.com/matrix-org/synapse/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/synapse/archive/master.tar.gz) | tar -xz --strip-components=1 -C /src/"
+          (wget -O - https://github.com/matrix-org/synapse/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/synapse/archive/master.tar.gz) \
+            | tar -xz --strip-components=1 -C /src/
 
       - name: Run sytest
         run: bash -xe /bootstrap.sh synapse

From cfda89b0141501ffa9f8ee9bd5f168ec525520e5 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 14:31:28 +0100
Subject: [PATCH 46/60] Fix branches and apply to dendrite

---
 .github/workflows/pipeline.yml | 36 +++++-----------------------------
 1 file changed, 5 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 22981a403..3ff3a2bae 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -56,7 +56,7 @@ jobs:
         shell: bash
         run: |
           BRANCH=${GITHUB_REF#refs/heads/}
-          (wget -O - https://github.com/matrix-org/synapse/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/synapse/archive/master.tar.gz) \
+          (wget -O - https://github.com/matrix-org/synapse/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/synapse/archive/develop.tar.gz) \
             | tar -xz --strip-components=1 -C /src/
 
       - name: Run sytest
@@ -102,38 +102,12 @@ jobs:
         with:
           path: sytest
 
-      # From https://stackoverflow.com/a/58035262
-      - name: Extract branch name
-        shell: bash
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: extract_branch
-
-      - name: Find dendrite ${{ steps.extract_branch.outputs.branch }} branch
-        id: find_dendrite_branch
-        uses: octokit/request-action@v2.1.0
-        with:
-          route: GET /repos/matrix-org/dendrite/branches/${{ steps.extract_branch.outputs.branch }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-
-      - name: Select dendrite branch to checkout
-        id: select_dendrite_branch
+      - name: Fetch corresponding synapse branch
         shell: bash
         run: |
-          if [ ${{ steps.find_dendrite_branch.outcome }} == "success" ]; then
-            branch=${{ steps.extract_branch.outputs.branch }}
-          else
-            branch=master
-          fi
-          echo "::set-output name=branch::$branch"
-
-      - name: Checkout dendrite ${{ steps.select_dendrite_branch.outputs.branch }} branch
-        uses: actions/checkout@v2
-        with:
-          repository: matrix-org/dendrite
-          path: dendrite
-          ref: ${{ steps.select_dendrite_branch.outputs.branch }}
+          BRANCH=${GITHUB_REF#refs/heads/}
+          (wget -O - https://github.com/matrix-org/dendrite/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/dendrite/archive/master.tar.gz) \
+            | tar -xz --strip-components=1 -C /src/
 
       - name: Run sytest
         run: bash -xe /bootstrap.sh dendrite

From 0dc6a18104e4c40795f5096f3e9668d62cc46c60 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 14:54:55 +0100
Subject: [PATCH 47/60] Oh, workers was an additional blacklist, not a
 replacement

---
 .github/workflows/pipeline.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 3ff3a2bae..3f5171114 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -43,7 +43,7 @@ jobs:
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
         WORKERS: ${{ matrix.synapse-workers && 1 }}
-        BLACKLIST: ${{ (matrix.workers && '.ci/worker-blacklist') || 'sytest-blacklist' }}
+        BLACKLIST: ${{ (matrix.workers && 'synapse-blacklist-with-workers') || 'sytest-blacklist' }}
         API: ${{ matrix.dendrite-full-http && 1 }}
 
     steps:
@@ -59,6 +59,10 @@ jobs:
           (wget -O - https://github.com/matrix-org/synapse/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/synapse/archive/develop.tar.gz) \
             | tar -xz --strip-components=1 -C /src/
 
+      - name: Prepare blacklist file
+        if: ${{ matrix.workers }}
+        run: cat /src/sytest-blacklist /src/.ci/worker-blacklist > /src/synapse-blacklist-with-workers
+
       - name: Run sytest
         run: bash -xe /bootstrap.sh synapse
 

From 03a59bf8288c1c8e155ff73d2199b91df908ea6f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 12 Aug 2021 15:13:39 +0100
Subject: [PATCH 48/60] Don't special case the github-actions branch

was just to force actions to trigger in my fork.
---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 3f5171114..87dbff979 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -1,7 +1,7 @@
 name: Run sytest
 on:
   push:
-    branches: ["develop", "release-*", "github-actions"]
+    branches: ["develop", "release-*"]
   pull_request:
 
 concurrency:

From fd51fc4e39d53eaeeb2195c9dc373657280c6647 Mon Sep 17 00:00:00 2001
From: David Robertson <david.m.robertson1@gmail.com>
Date: Fri, 13 Aug 2021 14:25:06 +0100
Subject: [PATCH 49/60] Fix "fetch dendrite branch" step name

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 87dbff979..e191f7b84 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -106,7 +106,7 @@ jobs:
         with:
           path: sytest
 
-      - name: Fetch corresponding synapse branch
+      - name: Fetch corresponding dendrite branch
         shell: bash
         run: |
           BRANCH=${GITHUB_REF#refs/heads/}

From 2984f8f0e719d96a7ff2858cbc12774c965def20 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 13 Aug 2021 14:36:51 +0100
Subject: [PATCH 50/60] Add explanatory comments

---
 .github/workflows/pipeline.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index e191f7b84..0592400ed 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -4,6 +4,9 @@ on:
     branches: ["develop", "release-*"]
   pull_request:
 
+# Only run this action once per pull request/branch; restart if a new commit arrives.
+# C.f. https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency
+# and https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -67,6 +70,7 @@ jobs:
         run: bash -xe /bootstrap.sh synapse
 
       - name: Summarise results.tap
+        # Use always() to run this step even if previous ones failed.
         if: ${{ always() }}
         run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
 

From 9e4a8f33fae3f191e0366e87ebb7e2f219c6be2e Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 13 Aug 2021 14:49:45 +0100
Subject: [PATCH 51/60] Try to get friendly job names

---
 .github/workflows/pipeline.yml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 0592400ed..66ffb6262 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -13,25 +13,31 @@ concurrency:
 
 jobs:
   synapse:
+    name: Synapse - ${{ matrix.job-name }}
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: bionic
+          - label: Python 3.6 / SQLite / Monolith
+            sytest-tag: bionic
 
-          - sytest-tag: bionic
+          - label: Python 3.6 / Postgres 10 / Monolith
+            sytest-tag: bionic
             postgres: postgres
 
-          - sytest-tag: bionic
+          - label: Python 3.6 / Postgres 10 / Workers
+            sytest-tag: bionic
             postgres: postgres
             workers: workers
 
-          - sytest-tag: testing
+          - label: Python 3.9 / Postgres 13 / Monolith
+            sytest-tag: testing
             postgres: postgres
 
-          - sytest-tag: testing
+          - label: Python 3.9 / Postgres 13 / Workers
+            sytest-tag: testing
             postgres: postgres
             workers: workers
 
@@ -85,7 +91,7 @@ jobs:
 
   dendrite:
     runs-on: ubuntu-latest
-
+    name: Dendrite - ${{ matrix.db }} ${{ (matrix.api == 'full-http') && 'full HTTP APIs' }}
     strategy:
       fail-fast: false
       matrix:

From b5350ec11faad12e5fcd336bf8a1472db8137f5e Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 13 Aug 2021 14:49:52 +0100
Subject: [PATCH 52/60] TEMP: hack in github-actions branch again

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 66ffb6262..4f5acfeb5 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -1,7 +1,7 @@
 name: Run sytest
 on:
   push:
-    branches: ["develop", "release-*"]
+    branches: ["develop", "release-*", "github-actions"]
   pull_request:
 
 # Only run this action once per pull request/branch; restart if a new commit arrives.

From dd7b896e54a0efae7faf6c2e13a5c326a7cd546f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 13 Aug 2021 15:07:51 +0100
Subject: [PATCH 53/60] Fiddle with labels; include, `&& 1` for dendrite

Need to ensure that API and POSTGRES are the empty strings for dendrite
when they're meant to be off. But ${{ matrix.api == 'full-http' && foo
}} seems to yield either foo or `false`. Stick with what we know works.
---
 .github/workflows/pipeline.yml | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 4f5acfeb5..891569939 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   synapse:
-    name: Synapse - ${{ matrix.job-name }}
+    name: Synapse - ${{ matrix.label }}
     runs-on: ubuntu-latest
 
     strategy:
@@ -84,19 +84,29 @@ jobs:
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Sytest Logs - ${{ job.status }} - (Synapse, ${{ join(matrix.*, ', ') }})
+          name: Sytest Logs - ${{ job.status }} - (Synapse, ${{ matrix.label }})
           path: |
             /logs/results.tap
             /logs/**/*.log*
 
   dendrite:
     runs-on: ubuntu-latest
-    name: Dendrite - ${{ matrix.db }} ${{ (matrix.api == 'full-http') && 'full HTTP APIs' }}
+    name: Dendrite - ${{ matrix.label }}
     strategy:
       fail-fast: false
       matrix:
-        db: ["sqlite", "postgres"]
-        api: ["not-full-http", "full-http"]
+        include:
+          - label: "SQLite"
+
+          - label: "SQLite / full HTTP APIs"
+            api: full-http
+
+          - label: "Postgres"
+            postgres: postgres
+
+          - label: "Postgres / full HTTP APIs"
+            postgres: postgres
+            api: full-http
 
     container:
       image: matrixdotorg/sytest-dendrite
@@ -107,8 +117,8 @@ jobs:
         # synapse_sytest.sh expects a synapse checkout at /src
         - ${{ github.workspace }}/dendrite:/src
       env:
-        POSTGRES: ${{ matrix.db == 'postgres' }}
-        API: ${{ matrix.api == 'full-http' }}
+        POSTGRES: ${{ matrix.postgres && 1 }}
+        API: ${{ matrix.api && 1 }}
 
     steps:
       - name: Checkout sytest

From 06e00d5ba1df13335c03e34a9a410dad268ed791 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 13 Aug 2021 15:09:49 +0100
Subject: [PATCH 54/60] Mark annotate.md as bk-specific

---
 scripts/dendrite_sytest.sh | 1 +
 scripts/synapse_sytest.sh  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh
index 08d68fc76..16e49d75c 100755
--- a/scripts/dendrite_sytest.sh
+++ b/scripts/dendrite_sytest.sh
@@ -68,6 +68,7 @@ echo >&2 "--- Copying assets"
 # Copy out the logs
 rsync -r --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
 
+# Generate annotate.md This is Buildkite-specific.
 if [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation
     perl /sytest/scripts/format_tap.pl /logs/results.tap "$GITHUB_SHA" >/logs/annotate.md
diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index 7f31dbbfe..c08608a4c 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -201,6 +201,7 @@ rsync --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /log
 #export TOP=/src
 #/venv/bin/coverage combine
 
+# Generate annotate.md This is Buildkite-specific.
 if [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation
     perl /sytest/scripts/format_tap.pl /logs/results.tap "$GITHUB_SHA" >/logs/annotate.md

From 85898383e6cb1aa7483a6729089463fcc265d1c9 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 13 Aug 2021 15:12:33 +0100
Subject: [PATCH 55/60] Shorten synapse labels

---
 .github/workflows/pipeline.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 891569939..fd48f8ce7 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -20,23 +20,23 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - label: Python 3.6 / SQLite / Monolith
+          - label: Py3.6 / SQLite / Monolith
             sytest-tag: bionic
 
-          - label: Python 3.6 / Postgres 10 / Monolith
+          - label: Py3.6 / PGSQL 10 / Monolith
             sytest-tag: bionic
             postgres: postgres
 
-          - label: Python 3.6 / Postgres 10 / Workers
+          - label: Py3.6 / PGSQL10 / Workers
             sytest-tag: bionic
             postgres: postgres
             workers: workers
 
-          - label: Python 3.9 / Postgres 13 / Monolith
+          - label: Py3.9 / PGSQL13 / Monolith
             sytest-tag: testing
             postgres: postgres
 
-          - label: Python 3.9 / Postgres 13 / Workers
+          - label: Py3.9 / PGSQL13 / Workers
             sytest-tag: testing
             postgres: postgres
             workers: workers

From ea25d7a553b1a9a4216c0e999312345e63322035 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 13 Aug 2021 15:16:12 +0100
Subject: [PATCH 56/60] More label faffing

---
 .github/workflows/pipeline.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index fd48f8ce7..4a6e5d99a 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -13,30 +13,30 @@ concurrency:
 
 jobs:
   synapse:
-    name: Synapse - ${{ matrix.label }}
+    name: "Synapse: ${{ matrix.label }}"
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
         include:
-          - label: Py3.6 / SQLite / Monolith
+          - label: Py 3.6, SQLite, Monolith
             sytest-tag: bionic
 
-          - label: Py3.6 / PGSQL 10 / Monolith
+          - label: Py 3.6, PG 10, Monolith
             sytest-tag: bionic
             postgres: postgres
 
-          - label: Py3.6 / PGSQL10 / Workers
+          - label: Py 3.6, PG 10, Workers
             sytest-tag: bionic
             postgres: postgres
             workers: workers
 
-          - label: Py3.9 / PGSQL13 / Monolith
+          - label: Py 3.9, PG 13, Monolith
             sytest-tag: testing
             postgres: postgres
 
-          - label: Py3.9 / PGSQL13 / Workers
+          - label: Py 3.9, PG 13, Workers
             sytest-tag: testing
             postgres: postgres
             workers: workers
@@ -91,20 +91,20 @@ jobs:
 
   dendrite:
     runs-on: ubuntu-latest
-    name: Dendrite - ${{ matrix.label }}
+    name: "Dendrite: ${{ matrix.label }}"
     strategy:
       fail-fast: false
       matrix:
         include:
-          - label: "SQLite"
+          - label: SQLite
 
-          - label: "SQLite / full HTTP APIs"
+          - label: SQLite, full HTTP APIs
             api: full-http
 
-          - label: "Postgres"
+          - label: Postgres
             postgres: postgres
 
-          - label: "Postgres / full HTTP APIs"
+          - label: Postgres, full HTTP APIs
             postgres: postgres
             api: full-http
 

From 7a9ca4bbba2f6608b7d1167e3753fc73f134e55d Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 13 Aug 2021 15:33:07 +0100
Subject: [PATCH 57/60] Use BUILDKITE_LABEL for annotations again

but only on BUILDKITE. Keep this around until we're confident with the
GHA workflow.
---
 scripts/dendrite_sytest.sh | 4 ++--
 scripts/synapse_sytest.sh  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh
index 16e49d75c..3ce79040d 100755
--- a/scripts/dendrite_sytest.sh
+++ b/scripts/dendrite_sytest.sh
@@ -69,9 +69,9 @@ echo >&2 "--- Copying assets"
 rsync -r --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
 
 # Generate annotate.md This is Buildkite-specific.
-if [ $TEST_STATUS -ne 0 ]; then
+if [ -n "$BUILDKITE_LABEL" ] && [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation
-    perl /sytest/scripts/format_tap.pl /logs/results.tap "$GITHUB_SHA" >/logs/annotate.md
+    perl /sytest/scripts/format_tap.pl /logs/results.tap "$BUILDKITE_LABEL" >/logs/annotate.md
     # If show-expected-fail-tests logged something, put it into the annotation
     # Annotations from a failed build show at the top of buildkite, alerting
     # developers quickly as to what needs to change in the black/whitelist.
diff --git a/scripts/synapse_sytest.sh b/scripts/synapse_sytest.sh
index c08608a4c..d38227306 100755
--- a/scripts/synapse_sytest.sh
+++ b/scripts/synapse_sytest.sh
@@ -202,9 +202,9 @@ rsync --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /log
 #/venv/bin/coverage combine
 
 # Generate annotate.md This is Buildkite-specific.
-if [ $TEST_STATUS -ne 0 ]; then
+if [ -n "$BUILDKITE_LABEL" ] && [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation
-    perl /sytest/scripts/format_tap.pl /logs/results.tap "$GITHUB_SHA" >/logs/annotate.md
+    perl /sytest/scripts/format_tap.pl /logs/results.tap "$BUILDKITE_LABEL" >/logs/annotate.md
 fi
 
 exit $TEST_STATUS

From 0994d0c5046d738d4197c5a01ee0dc20556f6ce0 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 18 Aug 2021 13:00:12 +0100
Subject: [PATCH 58/60] Fix and improve handling of env vars

- Don't use API for synapse; use right matrix key for WORKERS
- echo env vars before we invoke bootstrap script
---
 .github/workflows/pipeline.yml | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 4a6e5d99a..9ec19c7f7 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -51,9 +51,8 @@ jobs:
         - ${{ github.workspace }}/synapse:/src
       env:
         POSTGRES: ${{ matrix.postgres && 1 }}
-        WORKERS: ${{ matrix.synapse-workers && 1 }}
+        WORKERS: ${{ matrix.workers && 1 }}
         BLACKLIST: ${{ (matrix.workers && 'synapse-blacklist-with-workers') || 'sytest-blacklist' }}
-        API: ${{ matrix.dendrite-full-http && 1 }}
 
     steps:
       - name: Checkout sytest
@@ -73,7 +72,11 @@ jobs:
         run: cat /src/sytest-blacklist /src/.ci/worker-blacklist > /src/synapse-blacklist-with-workers
 
       - name: Run sytest
-        run: bash -xe /bootstrap.sh synapse
+        run: |
+          echo POSTGRES=${POSTGRES:-<NOT SET>}
+          echo WORKERS=${WORKERS:-<NOT SET>}
+          echo BLACKLIST=${BLACKLIST:-<NOT SET>}
+          bash -xe /bootstrap.sh synapse
 
       - name: Summarise results.tap
         # Use always() to run this step even if previous ones failed.
@@ -134,7 +137,10 @@ jobs:
             | tar -xz --strip-components=1 -C /src/
 
       - name: Run sytest
-        run: bash -xe /bootstrap.sh dendrite
+        run: |
+          echo POSTGRES=${POSTGRES:-<NOT SET>}
+          echo API=${API:-<NOT SET>}
+          bash -xe /bootstrap.sh dendrite
 
       - name: Summarise results.tap
         if: ${{ always() }}

From ce29fb2ad8a08715210587eb768b3e55867065b4 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 18 Aug 2021 13:24:54 +0100
Subject: [PATCH 59/60] More descriptive blacklist step name

---
 .github/workflows/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 9ec19c7f7..cfcc4f173 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -67,7 +67,7 @@ jobs:
           (wget -O - https://github.com/matrix-org/synapse/archive/$BRANCH.tar.gz || wget -O - https://github.com/matrix-org/synapse/archive/develop.tar.gz) \
             | tar -xz --strip-components=1 -C /src/
 
-      - name: Prepare blacklist file
+      - name: Prepare blacklist file for running with workers
         if: ${{ matrix.workers }}
         run: cat /src/sytest-blacklist /src/.ci/worker-blacklist > /src/synapse-blacklist-with-workers
 

From 5692410a85154dc0e5363dc12678f28d7e29ad7c Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 19 Aug 2021 14:28:02 +0100
Subject: [PATCH 60/60] We do actually want CI to pass a branch

---
 docker/bootstrap.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh
index e0ec9e4f5..836f3da3d 100755
--- a/docker/bootstrap.sh
+++ b/docker/bootstrap.sh
@@ -13,10 +13,14 @@ if [ -d "/sytest" ]; then
     echo "Using local sytests"
 else
     echo "--- Trying to get same-named sytest branch..."
-    # Otherwise, try and find the branch that the Synapse/Dendrite checkout
-    # is using. Fall back to develop if unknown.
-    branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
-
+    # Check if we're running in CI. If so it can tell us what
+    # Synapse/Dendrite branch we're running
+    if [ -n "$SYTEST_BRANCH" ]; then
+        branch_name="$SYTEST_BRANCH"
+    else
+        # Otherwise, try and find the branch that the Synapse/Dendrite checkout
+        # is using. Fall back to develop if unknown.
+        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
     if [ "$SYTEST_TARGET" == "dendrite" ] && [ "$branch_name" == "master" ]; then
         # Dendrite uses master as its main branch. If the branch is master, we probably want sytest develop
         branch_name="develop"