diff --git a/.github/workflows/cluster_endtoend_11.yml b/.github/workflows/cluster_endtoend_11.yml
new file mode 100644
index 00000000000..3919a50e2e6
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_11.yml
@@ -0,0 +1,37 @@
+name: Cluster (11)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 11
diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml
new file mode 100644
index 00000000000..ab75cabe6ce
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_12.yml
@@ -0,0 +1,37 @@
+name: Cluster (12)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 12
diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml
new file mode 100644
index 00000000000..926f16979d5
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_13.yml
@@ -0,0 +1,37 @@
+name: Cluster (13)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 13
diff --git a/.github/workflows/cluster_endtoend_14.yml b/.github/workflows/cluster_endtoend_14.yml
new file mode 100644
index 00000000000..ca182cc9791
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_14.yml
@@ -0,0 +1,37 @@
+name: Cluster (14)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 14
diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml
new file mode 100644
index 00000000000..d5be40668c1
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_15.yml
@@ -0,0 +1,37 @@
+name: Cluster (15)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 15
diff --git a/.github/workflows/cluster_endtoend_16.yml b/.github/workflows/cluster_endtoend_16.yml
new file mode 100644
index 00000000000..ce6b0c8b0a0
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_16.yml
@@ -0,0 +1,37 @@
+name: Cluster (16)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 16
diff --git a/.github/workflows/cluster_endtoend_17.yml b/.github/workflows/cluster_endtoend_17.yml
new file mode 100644
index 00000000000..cb1870437c5
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_17.yml
@@ -0,0 +1,37 @@
+name: Cluster (17)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 17
diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend_18.yml
similarity index 74%
rename from .github/workflows/cluster_endtoend.yml
rename to .github/workflows/cluster_endtoend_18.yml
index 60edd6e3405..4759d8b9c94 100644
--- a/.github/workflows/cluster_endtoend.yml
+++ b/.github/workflows/cluster_endtoend_18.yml
@@ -1,12 +1,9 @@
-name: cluster_endtoend
+name: Cluster (18)
 on: [push, pull_request]
 jobs:
 
   build:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 27]
 
     steps:
     - name: Set up Go
@@ -26,6 +23,7 @@ jobs:
         sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
+
         wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
@@ -34,14 +32,10 @@ jobs:
 
     - name: Installing zookeeper and consul
       run: |
-        # Only running for shard 18 and 24 where we need to install consul and zookeeper
-        if [[ ${{matrix.name}} == 18 || ${{matrix.name}} == 24 ]]; then
           make tools
-        fi
 
-    - name: sharded cluster_endtoend
+    - name: Run cluster endtoend test
       timeout-minutes: 30
       run: |
         source build.env
-        eatmydata -- go run test.go -docker=false -print-log -follow -shard ${{matrix.name}}
-        
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 18
diff --git a/.github/workflows/cluster_endtoend_19.yml b/.github/workflows/cluster_endtoend_19.yml
new file mode 100644
index 00000000000..facafc4c12c
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_19.yml
@@ -0,0 +1,37 @@
+name: Cluster (19)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 19
diff --git a/.github/workflows/cluster_endtoend_20.yml b/.github/workflows/cluster_endtoend_20.yml
new file mode 100644
index 00000000000..f440a00e305
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_20.yml
@@ -0,0 +1,37 @@
+name: Cluster (20)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 20
diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml
new file mode 100644
index 00000000000..3f601ebf2cc
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_21.yml
@@ -0,0 +1,37 @@
+name: Cluster (21)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 21
diff --git a/.github/workflows/cluster_endtoend_22.yml b/.github/workflows/cluster_endtoend_22.yml
new file mode 100644
index 00000000000..9d6d5ac299b
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_22.yml
@@ -0,0 +1,37 @@
+name: Cluster (22)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 22
diff --git a/.github/workflows/cluster_endtoend_23.yml b/.github/workflows/cluster_endtoend_23.yml
new file mode 100644
index 00000000000..c6185df359a
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_23.yml
@@ -0,0 +1,37 @@
+name: Cluster (23)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 23
diff --git a/.github/workflows/cluster_endtoend_24.yml b/.github/workflows/cluster_endtoend_24.yml
new file mode 100644
index 00000000000..cf09d8825b5
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_24.yml
@@ -0,0 +1,41 @@
+name: Cluster (24)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Installing zookeeper and consul
+      run: |
+          make tools
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 24
diff --git a/.github/workflows/cluster_endtoend_26.yml b/.github/workflows/cluster_endtoend_26.yml
new file mode 100644
index 00000000000..f59c891e2c9
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_26.yml
@@ -0,0 +1,37 @@
+name: Cluster (26)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 26
diff --git a/.github/workflows/cluster_endtoend_27.yml b/.github/workflows/cluster_endtoend_27.yml
new file mode 100644
index 00000000000..f51b0594346
--- /dev/null
+++ b/.github/workflows/cluster_endtoend_27.yml
@@ -0,0 +1,37 @@
+name: Cluster (27)
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 27
diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml
deleted file mode 100644
index ba58c5af400..00000000000
--- a/.github/workflows/unit.yml
+++ /dev/null
@@ -1,90 +0,0 @@
-name: unit
-on: [push, pull_request]
-jobs:
-
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        name: [percona56, mysql57, mysql80, mariadb101, mariadb102, mariadb103]
-
-    steps:
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.15
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Get dependencies
-      run: |
-        export DEBIAN_FRONTEND="noninteractive"
-        sudo apt-get update
-
-        if [ ${{matrix.name}} = "mysql57" ]; then
-          sudo apt-get install -y mysql-server mysql-client
-        else
-          # Uninstall likely installed MySQL first
-          sudo systemctl stop apparmor
-          sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
-          sudo apt-get -y autoremove
-          sudo apt-get -y autoclean
-          sudo deluser mysql
-          sudo rm -rf /var/lib/mysql
-          sudo rm -rf /etc/mysql
- 
-          if [ ${{matrix.name}} = "percona56" ]; then
-            sudo rm -rf /var/lib/mysql
-            sudo apt install -y gnupg2
-            wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
-            sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
-            sudo apt update
-            sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y percona-server-server-5.6 percona-server-client-5.6  
-          elif [ ${{matrix.name}} = "mysql80" ]; then
-            wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb
-            echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections 
-            sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
-            sudo apt-get update
-            sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
-          elif [ ${{matrix.name}} = "mariadb101" ]; then
-            sudo apt-get install -y software-properties-common
-            sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
-            sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu bionic main'
-            sudo apt update
-            sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
-          elif [ ${{matrix.name}} = "mariadb102" ]; then
-            sudo apt-get install -y software-properties-common
-            sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
-            sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.2/ubuntu bionic main'
-            sudo apt update
-            sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
-          elif [ ${{matrix.name}} = "mariadb103" ]; then
-            sudo apt-get install -y software-properties-common
-            sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
-            sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main'
-            sudo apt update
-            sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
-          fi
-        fi
-
-        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk eatmydata
-        sudo service mysql stop
-        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
-        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
-
-        mkdir -p dist bin
-        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
-        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
-
-        go mod download
-
-    - name: Run make tools
-      run: |
-        make tools
-
-    - name: unit
-      timeout-minutes: 30
-      run: |
-        eatmydata -- make unit_test
diff --git a/.github/workflows/unit_test_mariadb101.yml b/.github/workflows/unit_test_mariadb101.yml
new file mode 100644
index 00000000000..56efe312a0f
--- /dev/null
+++ b/.github/workflows/unit_test_mariadb101.yml
@@ -0,0 +1,59 @@
+name: Unit Test (mariadb101)
+on: [push, pull_request]
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        export DEBIAN_FRONTEND="noninteractive"
+        sudo apt-get update
+
+        # !mysql57
+
+        # Uninstall any previously installed MySQL first
+        sudo systemctl stop apparmor
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
+        sudo apt-get -y autoremove
+        sudo apt-get -y autoclean
+        sudo deluser mysql
+        sudo rm -rf /var/lib/mysql
+        sudo rm -rf /etc/mysql
+
+        # mariadb101
+        sudo apt-get install -y software-properties-common
+        sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+        sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu bionic main'
+        sudo apt update
+        sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
+
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk eatmydata
+        sudo service mysql stop
+        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
+
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
+        go mod download
+
+    - name: Run make tools
+      run: |
+        make tools
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        eatmydata -- make unit_test
diff --git a/.github/workflows/unit_test_mariadb102.yml b/.github/workflows/unit_test_mariadb102.yml
new file mode 100644
index 00000000000..0e8eb1f5e75
--- /dev/null
+++ b/.github/workflows/unit_test_mariadb102.yml
@@ -0,0 +1,59 @@
+name: Unit Test (mariadb102)
+on: [push, pull_request]
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        export DEBIAN_FRONTEND="noninteractive"
+        sudo apt-get update
+
+        # !mysql57
+
+        # Uninstall any previously installed MySQL first
+        sudo systemctl stop apparmor
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
+        sudo apt-get -y autoremove
+        sudo apt-get -y autoclean
+        sudo deluser mysql
+        sudo rm -rf /var/lib/mysql
+        sudo rm -rf /etc/mysql
+
+        # mariadb102
+        sudo apt-get install -y software-properties-common
+        sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+        sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.2/ubuntu bionic main'
+        sudo apt update
+        sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
+
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk eatmydata
+        sudo service mysql stop
+        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
+
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
+        go mod download
+
+    - name: Run make tools
+      run: |
+        make tools
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        eatmydata -- make unit_test
diff --git a/.github/workflows/unit_test_mariadb103.yml b/.github/workflows/unit_test_mariadb103.yml
new file mode 100644
index 00000000000..9ce9ab129fa
--- /dev/null
+++ b/.github/workflows/unit_test_mariadb103.yml
@@ -0,0 +1,59 @@
+name: Unit Test (mariadb103)
+on: [push, pull_request]
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        export DEBIAN_FRONTEND="noninteractive"
+        sudo apt-get update
+
+        # !mysql57
+
+        # Uninstall any previously installed MySQL first
+        sudo systemctl stop apparmor
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
+        sudo apt-get -y autoremove
+        sudo apt-get -y autoclean
+        sudo deluser mysql
+        sudo rm -rf /var/lib/mysql
+        sudo rm -rf /etc/mysql
+
+        # mariadb103
+        sudo apt-get install -y software-properties-common
+        sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+        sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main'
+        sudo apt update
+        sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
+
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk eatmydata
+        sudo service mysql stop
+        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
+
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
+        go mod download
+
+    - name: Run make tools
+      run: |
+        make tools
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        eatmydata -- make unit_test
diff --git a/.github/workflows/unit_test_mysql57.yml b/.github/workflows/unit_test_mysql57.yml
new file mode 100644
index 00000000000..971c4f11b7e
--- /dev/null
+++ b/.github/workflows/unit_test_mysql57.yml
@@ -0,0 +1,44 @@
+name: Unit Test (mysql57)
+on: [push, pull_request]
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        export DEBIAN_FRONTEND="noninteractive"
+        sudo apt-get update
+
+        # mysql57
+        sudo apt-get install -y mysql-server mysql-client
+
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk eatmydata
+        sudo service mysql stop
+        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
+
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
+        go mod download
+
+    - name: Run make tools
+      run: |
+        make tools
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        eatmydata -- make unit_test
diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml
new file mode 100644
index 00000000000..a4002bed5b5
--- /dev/null
+++ b/.github/workflows/unit_test_mysql80.yml
@@ -0,0 +1,59 @@
+name: Unit Test (mysql80)
+on: [push, pull_request]
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        export DEBIAN_FRONTEND="noninteractive"
+        sudo apt-get update
+
+        # !mysql57
+
+        # Uninstall any previously installed MySQL first
+        sudo systemctl stop apparmor
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
+        sudo apt-get -y autoremove
+        sudo apt-get -y autoclean
+        sudo deluser mysql
+        sudo rm -rf /var/lib/mysql
+        sudo rm -rf /etc/mysql
+
+        # mysql80
+        wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb
+        echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
+        sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
+        sudo apt-get update
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
+
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk eatmydata
+        sudo service mysql stop
+        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
+
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
+        go mod download
+
+    - name: Run make tools
+      run: |
+        make tools
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        eatmydata -- make unit_test
diff --git a/.github/workflows/unit_test_percona56.yml b/.github/workflows/unit_test_percona56.yml
new file mode 100644
index 00000000000..0b5dc44be1e
--- /dev/null
+++ b/.github/workflows/unit_test_percona56.yml
@@ -0,0 +1,60 @@
+name: Unit Test (percona56)
+on: [push, pull_request]
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        export DEBIAN_FRONTEND="noninteractive"
+        sudo apt-get update
+
+        # !mysql57
+
+        # Uninstall any previously installed MySQL first
+        sudo systemctl stop apparmor
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
+        sudo apt-get -y autoremove
+        sudo apt-get -y autoclean
+        sudo deluser mysql
+        sudo rm -rf /var/lib/mysql
+        sudo rm -rf /etc/mysql
+
+        # percona56
+        sudo rm -rf /var/lib/mysql
+        sudo apt install -y gnupg2
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt update
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y percona-server-server-5.6 percona-server-client-5.6
+
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk eatmydata
+        sudo service mysql stop
+        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
+
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
+        go mod download
+
+    - name: Run make tools
+      run: |
+        make tools
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        eatmydata -- make unit_test
diff --git a/Makefile b/Makefile
index 1d65038406f..553c1fc8be9 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ export REWRITER=go/vt/sqlparser/rewriter.go
 # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time.
 .NOTPARALLEL:
 
-.PHONY: all build install test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools web_bootstrap web_build web_start
+.PHONY: all build install test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools web_bootstrap web_build web_start generate_ci_workflows
 
 all: build
 
@@ -325,3 +325,9 @@ vtadmin_web_install:
 # While vtadmin-web is new and unstable, however, we can keep it out of the critical build path.
 vtadmin_web_proto_types: vtadmin_web_install
 	./web/vtadmin/bin/generate-proto-types.sh
+
+# Generate github CI actions workflow files for unit tests and cluster endtoend tests based on templates in the test/templates directory
+# Needs to be called if the templates change or if a new test "shard" is created. We do not need to rebuild tests if only the test/config.json
+# is changed by adding a new test to an existing shard. Any new or modified files need to be committed into git
+generate_ci_workflows:
+	cd test && go run ci_workflow_gen.go && cd ..
diff --git a/test/README.md b/test/README.md
new file mode 100644
index 00000000000..a1e0afe8782
--- /dev/null
+++ b/test/README.md
@@ -0,0 +1,44 @@
+##Github CI Workflows
+
+This document has a short outline of how tests are run in CI, how to add new tests and where these are configured.
+
+### Adding a new test
+
+Unit tests are run by the unit test runner, one per platform, currently percona56, mysql57, mysql80, mariadb101, mariadb102, mariadb103.
+The workflow first installs the required database server before calling `make unit_test`.
+
+To add a new end-to-end (e2e) test (also called _cluster end to end_ tests):
+* Add a new object to test/config.json
+* If you are creating a new test _shard_:
+  * update `clusterList` in `ci_workflow_gen.go`
+  * `make generate_ci_workflows`
+* If you are adding a new database platform, update the `templates\unit_test.tpl` to add 
+  the platform specific packages and update `unitTestDatabases`
+
+
+### Vitess test runner
+The `.github/workflows` directory contains one yaml file per workflow. e2e tests are run using the `test.go` script 
+in the repository root.
+
+This script invokes the vitess e2e test framework using a json configuration file `test/config.json` which has one object per test. 
+Each test is of the form:
+
+```javascript
+"vtgate": {
+			"File": "unused.go",
+			"Args": ["vitess.io/vitess/go/test/endtoend/vtgate"],
+			"Command": [],
+			"Manual": false,
+			"Shard": 17,
+			"RetryMax": 0,
+			"Tags": []
+		},
+```
+The important parameters here are Args which define the arguments to `go test` and the Shard which says 
+which Test VM should run this test. All tests which have a common Shard value are run in the same test vm.
+
+### Known Issue
+
+* Each VM does not seem to be able to create a lot of vttablets. For this reason we have had to split a few VReplication 
+e2e tests across Shards. We need to identify and if possible fix this limitation so that we can reduce the number of test Shards
+  
\ No newline at end of file
diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go
new file mode 100644
index 00000000000..c32a51529b2
--- /dev/null
+++ b/test/ci_workflow_gen.go
@@ -0,0 +1,145 @@
+/*
+Copyright 2021 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"text/template"
+)
+
+const (
+	workflowConfigDir = "../.github/workflows"
+
+	unitTestTemplate  = "templates/unit_test.tpl"
+	unitTestDatabases = "percona56, mysql57, mysql80, mariadb101, mariadb102, mariadb103"
+
+	clusterTestTemplate        = "templates/cluster_endtoend_test.tpl"
+	clusterList                = "11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 27"
+	clustersRequiringMakeTools = "18,24"
+
+	// TODO: currently some percona tools including xtrabackup are installed on all clusters, we can possibly optimize
+	// this by only installing them in the required clusters
+	clustersRequiringXtraBackup = clusterList
+)
+
+type unitTest struct {
+	Name, Database string
+}
+
+type clusterTest struct {
+	Name, Shard                  string
+	MakeTools, InstallXtraBackup bool
+}
+
+func mergeBlankLines(buf *bytes.Buffer) string {
+	var out []string
+	in := strings.Split(buf.String(), "\n")
+	lastWasBlank := false
+	for _, line := range in {
+		if strings.TrimSpace(line) == "" {
+			if lastWasBlank {
+				continue
+			}
+			lastWasBlank = true
+		} else {
+			lastWasBlank = false
+		}
+
+		out = append(out, line)
+	}
+	return strings.Join(out, "\n")
+}
+
+func main() {
+	generateUnitTestWorkflows()
+	generateClusterWorkflows()
+}
+
+func parseList(csvList string) []string {
+	var list []string
+	for _, item := range strings.Split(csvList, ",") {
+		list = append(list, strings.TrimSpace(item))
+	}
+	return list
+}
+
+func generateClusterWorkflows() {
+	clusters := parseList(clusterList)
+	for _, cluster := range clusters {
+		test := &clusterTest{
+			Name:  fmt.Sprintf("Cluster (%s)", cluster),
+			Shard: cluster,
+		}
+		makeToolClusters := parseList(clustersRequiringMakeTools)
+		for _, makeToolCluster := range makeToolClusters {
+			if makeToolCluster == cluster {
+				test.MakeTools = true
+				break
+			}
+		}
+		xtraBackupClusters := parseList(clustersRequiringXtraBackup)
+		for _, xtraBackupCluster := range xtraBackupClusters {
+			if xtraBackupCluster == cluster {
+				test.InstallXtraBackup = true
+				break
+			}
+		}
+
+		path := fmt.Sprintf("%s/cluster_endtoend_%s.yml", workflowConfigDir, cluster)
+		generateWorkflowFile(clusterTestTemplate, path, test)
+	}
+}
+
+func generateUnitTestWorkflows() {
+	platforms := parseList(unitTestDatabases)
+	for _, platform := range platforms {
+		test := &unitTest{
+			Name:     fmt.Sprintf("Unit Test (%s)", platform),
+			Database: platform,
+		}
+		path := fmt.Sprintf("%s/unit_test_%s.yml", workflowConfigDir, platform)
+		generateWorkflowFile(unitTestTemplate, path, test)
+	}
+}
+
+func generateWorkflowFile(templateFile, path string, test interface{}) {
+	tpl, err := template.ParseFiles(templateFile)
+	if err != nil {
+		fmt.Printf("Error: %s\n", err)
+		return
+	}
+
+	buf := &bytes.Buffer{}
+	err = tpl.Execute(buf, test)
+	if err != nil {
+		fmt.Printf("Error: %s\n", err)
+		return
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		log.Println("Error creating file: ", err)
+		return
+	}
+	f.WriteString(mergeBlankLines(buf))
+	fmt.Printf("Generated %s\n", path)
+
+}
diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl
new file mode 100644
index 00000000000..b0ce7f42eab
--- /dev/null
+++ b/test/templates/cluster_endtoend_test.tpl
@@ -0,0 +1,49 @@
+name: {{.Name}}
+on: [push, pull_request]
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        {{if .InstallXtraBackup}}
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+        {{end}}
+
+    {{if .MakeTools}}
+
+    - name: Installing zookeeper and consul
+      run: |
+          make tools
+
+    {{end}}
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard {{.Shard}}
diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl
new file mode 100644
index 00000000000..967e8c86eac
--- /dev/null
+++ b/test/templates/unit_test.tpl
@@ -0,0 +1,117 @@
+name: {{.Name}}
+on: [push, pull_request]
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        export DEBIAN_FRONTEND="noninteractive"
+        sudo apt-get update
+
+        {{if (eq .Platform "mysql57")}}
+
+        # mysql57
+        sudo apt-get install -y mysql-server mysql-client
+
+        {{else}}
+
+        # !mysql57
+
+        # Uninstall any previously installed MySQL first
+        sudo systemctl stop apparmor
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
+        sudo apt-get -y autoremove
+        sudo apt-get -y autoclean
+        sudo deluser mysql
+        sudo rm -rf /var/lib/mysql
+        sudo rm -rf /etc/mysql
+
+        {{if (eq .Platform "percona56")}}
+
+        # percona56
+        sudo rm -rf /var/lib/mysql
+        sudo apt install -y gnupg2
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt update
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y percona-server-server-5.6 percona-server-client-5.6
+
+        {{end}}
+
+        {{if (eq .Platform "mysql80")}}
+
+        # mysql80
+        wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb
+        echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
+        sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
+        sudo apt-get update
+        sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
+
+        {{end}}
+
+        {{if (eq .Platform "mariadb101")}}
+
+        # mariadb101
+        sudo apt-get install -y software-properties-common
+        sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+        sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu bionic main'
+        sudo apt update
+        sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
+
+        {{end}}
+
+        {{if (eq .Platform "mariadb102")}}
+
+        # mariadb102
+        sudo apt-get install -y software-properties-common
+        sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+        sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.2/ubuntu bionic main'
+        sudo apt update
+        sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
+
+        {{end}}
+
+        {{if (eq .Platform "mariadb103")}}
+
+        # mariadb103
+        sudo apt-get install -y software-properties-common
+        sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+        sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://nyc2.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main'
+        sudo apt update
+        sudo DEBIAN_FRONTEND="noninteractive" apt install -y mariadb-server
+
+        {{end}}
+
+        {{end}} {{/*outer if*/}}
+
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk eatmydata
+        sudo service mysql stop
+        sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
+
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
+
+        go mod download
+
+    - name: Run make tools
+      run: |
+        make tools
+
+    - name: Run test
+      timeout-minutes: 30
+      run: |
+        eatmydata -- make unit_test