Refactor vtorc endtoend tests

.github/workflows/cluster_endtoend_vtorc.yml

@@ -0,0 +1,50 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Cluster (vtorc)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (vtorc)
+    runs-on: ubuntu-18.04
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
+    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      run: |
+        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard vtorc

go/test/endtoend/orchestrator/orc_test.go → go/test/endtoend/vtorc/vtorc_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package orchestrator
+package vtorc
 
 import (
 	"context"
@@ -204,6 +204,24 @@ func TestDownMaster(t *testing.T) {
 	}
 }
 
+func waitForReadOnlyValue(t *testing.T, curMaster *cluster.Vttablet, expectValue int64) (match bool) {
+	timeout := 15 * time.Second
+	startTime := time.Now()
+	for time.Since(startTime) < timeout {
+		qr := runSQL(t, "select @@global.read_only as read_only", curMaster, "")
+		require.NotNil(t, qr)
+		row := qr.Named().Row()
+		require.NotNil(t, row)
+		readOnly, err := row.ToInt64("read_only")
+		require.NoError(t, err)
+		if readOnly == expectValue {
+			return true
+		}
+		time.Sleep(time.Second)
+	}
+	return false
+}
+
 // 3. make master readonly, let orc repair
 func TestMasterReadOnly(t *testing.T) {
 	defer cluster.PanicHandler(t)
@@ -224,12 +242,8 @@ func TestMasterReadOnly(t *testing.T) {
 	runSQL(t, "set global read_only=ON", curMaster, "")
 
 	// wait for repair
-	// TODO(deepthi): wait for condition instead of sleep
-	time.Sleep(15 * time.Second)
-	qr := runSQL(t, "select @@global.read_only", curMaster, "")
-	require.NotNil(t, qr)
-	require.Equal(t, 1, len(qr.Rows))
-	require.Equal(t, "[[INT64(0)]]", fmt.Sprintf("%s", qr.Rows), qr.Rows)
+	match := waitForReadOnlyValue(t, curMaster, 0)
+	require.True(t, match)
 }
 
 // 4. make replica ReadWrite, let orc repair
@@ -260,12 +274,8 @@ func TestReplicaReadWrite(t *testing.T) {
 	runSQL(t, "set global read_only=OFF", replica, "")
 
 	// wait for repair
-	// TODO(deepthi): wait for condition instead of sleep
-	time.Sleep(15 * time.Second)
-	qr := runSQL(t, "select @@global.read_only", replica, "")
-	require.NotNil(t, qr)
-	require.Equal(t, 1, len(qr.Rows))
-	require.Equal(t, "[[INT64(1)]]", fmt.Sprintf("%s", qr.Rows), qr.Rows)
+	match := waitForReadOnlyValue(t, replica, 1)
+	require.True(t, match)
 }
 
 // 5. stop replication, let orc repair

test/ci_workflow_gen.go

@@ -66,6 +66,7 @@ var (
 		"tabletmanager_throttler",
 		"tabletmanager_throttler_custom_config",
 		"tabletmanager_tablegc",
+		"vtorc",
 	}
 	// TODO: currently some percona tools including xtrabackup are installed on all clusters, we can possibly optimize
 	// this by only installing them in the required clusters

test/config.json

@@ -779,12 +779,12 @@
 			"RetryMax": 0,
 			"Tags": []
 		},
-		"orchestrator": {
+		"vtorc": {
 			"File": "unused.go",
-			"Args": ["vitess.io/vitess/go/test/endtoend/orchestrator"],
+			"Args": ["vitess.io/vitess/go/test/endtoend/vtorc"],
 			"Command": [],
 			"Manual": false,
-			"Shard": "22",
+			"Shard": "vtorc",
 			"RetryMax": 0,
 			"Tags": []
 		},