Merge pull request #9551 from planetscale/move-vtorc-back

Move vtorc back to normal runners
vitessio · Jan 22, 2022 · 9e529b3 · 9e529b3
2 parents 3bd0c51 + 0d240e1
commit 9e529b3
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 65 deletions.
diff --git a/.github/docker/cluster_test_vtorc/Dockerfile b/.github/docker/cluster_test_vtorc/Dockerfile
diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml
@@ -6,36 +6,77 @@ concurrency:
   group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtorc)')
   cancel-in-progress: true
 
+env:
+  LAUNCHABLE_ORGANIZATION: "vitess"
+  LAUNCHABLE_WORKSPACE: "vitess-app"
+  GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}"
+
 jobs:
   build:
     name: Run endtoend tests on Cluster (vtorc)
-    runs-on: self-hosted
+    runs-on: ubuntu-18.04
 
     steps:
-      - name: Check out code
-        uses: actions/checkout@v2
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+
+    - name: Set up python
+      uses: actions/setup-python@v2
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        # install JUnit report formatter
+        go get -u github.com/vitessio/go-junit-report@HEAD
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Setup launchable dependencies
+      run: |
+        # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up
+        pip3 install --user launchable~=1.0 > /dev/null
+
+        # verify that launchable setup is all correct.
+        launchable verify || true
 
-      - name: Build Docker Image
-        run: docker build -f ./.github/docker/cluster_test_vtorc/Dockerfile -t cluster_test_vtorc:$GITHUB_SHA  .
+        # Tell Launchable about the build you are producing and testing
+        launchable record build --name "$GITHUB_RUN_ID" --source .
 
-      - name: Run test
-        timeout-minutes: 30
-        run: docker run --name "cluster_test_vtorc_$GITHUB_SHA" cluster_test_vtorc:$GITHUB_SHA /bin/bash -c 'source build.env && go run test.go -keep-data=true -docker=false -print-log -follow -shard vtorc -- -- --keep-data=true'
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
 
-      - name: Print Volume Used
-        if: ${{ always() }}
-        run: |
-          docker inspect -f '{{ (index .Mounts 0).Name }}' cluster_test_vtorc_$GITHUB_SHA
+        set -x
 
-      - name: Cleanup Docker Volume
-        run: |
-          docker rm -v cluster_test_vtorc_$GITHUB_SHA
+        # run the tests however you normally do, then produce a JUnit XML file
+        eatmydata -- go run test.go -docker=false -follow -shard vtorc | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 
-      - name: Cleanup Docker Container
-        if: ${{ always() }}
-        run: |
-          docker rm -f cluster_test_vtorc_$GITHUB_SHA
+    - name: Print test output and Record test result in launchable
+      run: |
+        # send recorded tests to launchable
+        launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
 
-      - name: Cleanup Docker Image
-        run: |
-          docker image rm cluster_test_vtorc:$GITHUB_SHA
+        # print test output
+        cat output.txt
+      if: always()
diff --git a/GITHUB_SELF_HOSTED_RUNNERS.md b/GITHUB_SELF_HOSTED_RUNNERS.md
@@ -25,6 +25,12 @@ access to Vitess.
 8. Set up a cron job to remove docker volumes and images every week
    1. `crontab -e`
    2. Within the file add a line `8 5 * * 6 docker system prune -f --volumes --all`
+9. Vtorc, Cluster 14 and some other tests use multiple MySQL instances which are all brought up with asynchronous I/O setup in InnoDB. This sometimes leads to us hitting the Linux asynchronous I/O limit.
+To fix this we increase the default limit on the self-hosted runners by -
+   1. To set the aio-max-nr value, add the following line to the /etc/sysctl.conf file:
+      1. `fs.aio-max-nr = 1048576`
+   2. To activate the new setting, run the following command:
+      1. `sysctl -p /etc/sysctl.conf`
 
 ### Moving a test to a self-hosted runner
 Most of the code for running the tests is generated code by `make generate_ci_workflows` which uses the file `ci_workflow_gen.go`

diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go
@@ -102,11 +102,11 @@ var (
 		"mysql80",
 		"vreplication_multicell",
 		"vreplication_cellalias",
+		"vtorc",
 	}
 
 	clusterSelfHostedList = []string{
 		"14",
-		"vtorc",
 	}
 	clusterDockerList = []string{
 		"vreplication_basic",