Testing upgrade path from / downgrade path to v8.0.0

.github/workflows/cluster_endtoend_upgrade.yml

@@ -0,0 +1,105 @@
+name: Cluster (upgrade)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (upgrade)
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out v8.0.0
+      uses: actions/checkout@v2
+      with:
+        ref: v8.0.0
+
+    - name: Get dependencies
+      run: |
+        # This prepares general purpose binary dependencies
+        # as well as v8.0.0 specific go modules
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Building v8.0.0 binaries
+      timeout-minutes: 10
+      run: |
+        # We build v8.0.0 binaries and save them in a temporary location
+        source build.env
+        make build
+        mkdir -p /tmp/vitess-build-v8.0.0/
+        cp -R bin /tmp/vitess-build-v8.0.0/
+
+    - name: Check out HEAD
+      uses: actions/checkout@v2
+
+    - name: Run cluster endtoend test v8.0.0 (create cluster)
+      timeout-minutes: 5
+      run: |
+        # By checking out we deleted bin/ directory. We now restore our pre-built v8.0.0 binaries
+        cp -R /tmp/vitess-build-v8.0.0/bin .
+        # create the directory where we store test data; ensure it is empty:
+        rm -rf /tmp/vtdataroot
+        mkdir -p /tmp/vtdataroot
+        source build.env
+        # We pass -skip-build so that we use the v8.0.0 binaries, not HEAD binaries
+        eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -shard 28
+
+    - name: Check out HEAD
+      uses: actions/checkout@v2
+
+
+    - name: Building HEAD binaries
+      timeout-minutes: 10
+      run: |
+        go mod download
+
+        source build.env
+        make build
+        mkdir -p /tmp/vitess-build-head/
+        cp -R bin /tmp/vitess-build-head/
+
+    - name: Run cluster endtoend test HEAD based on v8.0.0 data (upgrade path)
+      timeout-minutes: 5
+      run: |
+        # /tmp/vtdataroot exists from previous test
+
+        source build.env
+        # We built HEAD binaries manually in previous step and there's no need for the test to build.
+        eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -shard 28
+
+    - name: Run cluster endtoend test HEAD (create cluster)
+      timeout-minutes: 5
+      run: |
+        # create the directory where we store test data; ensure it is empty:
+        rm -rf /tmp/vtdataroot
+        mkdir -p /tmp/vtdataroot
+
+        source build.env
+        # We still have the binaries from previous step. No need to build
+        eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -shard 28
+
+
+    - name: Run cluster endtoend test v8.0.0 based on HEAD data (downgrade path)
+      timeout-minutes: 5
+      run: |
+        # /tmp/vtdataroot exists from previous test
+        cp -R /tmp/vitess-build-v8.0.0/bin .
+
+        source build.env
+        # We again built manually and there's no need for the test to build.
+        eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -shard 28

go/test/endtoend/cluster/cluster_process.go

@@ -42,9 +42,12 @@ const (
 )
 
 var (
-	keepData   = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders")
-	topoFlavor = flag.String("topo-flavor", "etcd2", "choose a topo server from etcd2, zk2 or consul")
-	isCoverage = flag.Bool("is-coverage", false, "whether coverage is required")
+	keepData           = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders")
+	topoFlavor         = flag.String("topo-flavor", "etcd2", "choose a topo server from etcd2, zk2 or consul")
+	isCoverage         = flag.Bool("is-coverage", false, "whether coverage is required")
+	forceVTDATAROOT    = flag.String("force-vtdataroot", "", "force path for VTDATAROOT, which may already be populated")
+	forcePortStart     = flag.Int("force-port-start", 0, "force assigning ports based on this seed")
+	forceBaseTabletUID = flag.Int("force-base-tablet-uid", 0, "force assigning tablet ports based on this seed")
 )
 
 // LocalProcessCluster Testcases need to use this to iniate a cluster
@@ -57,6 +60,7 @@ type LocalProcessCluster struct {
 	TmpDirectory       string
 	OriginalVTDATAROOT string
 	CurrentVTDATAROOT  string
+	ReusingVTDATAROOT  bool
 
 	VtgateMySQLPort int
 	VtgateGrpcPort  int
@@ -192,10 +196,12 @@ func (cluster *LocalProcessCluster) StartTopo() (err error) {
 		}
 	}
 
-	cluster.VtctlProcess = *VtctlProcessInstance(cluster.TopoProcess.Port, cluster.Hostname)
-	if err = cluster.VtctlProcess.AddCellInfo(cluster.Cell); err != nil {
-		log.Error(err)
-		return
+	if !cluster.ReusingVTDATAROOT {
+		cluster.VtctlProcess = *VtctlProcessInstance(cluster.TopoProcess.Port, cluster.Hostname)
+		if err = cluster.VtctlProcess.AddCellInfo(cluster.Cell); err != nil {
+			log.Error(err)
+			return
+		}
 	}
 
 	cluster.VtctldProcess = *VtctldProcessInstance(cluster.GetAndReservePort(), cluster.GetAndReservePort(),
@@ -230,7 +236,9 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames
 	}
 
 	log.Infof("Starting keyspace: %v", keyspace.Name)
-	_ = cluster.VtctlProcess.CreateKeyspace(keyspace.Name)
+	if !cluster.ReusingVTDATAROOT {
+		_ = cluster.VtctlProcess.CreateKeyspace(keyspace.Name)
+	}
 	var mysqlctlProcessList []*exec.Cmd
 	for _, shardName := range shardNames {
 		shard := &Shard{
@@ -256,7 +264,7 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames
 			}
 			// Start Mysqlctl process
 			log.Infof("Starting mysqlctl for table uid %d, mysql port %d", tablet.TabletUID, tablet.MySQLPort)
-			tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, cluster.TmpDirectory)
+			tablet.MysqlctlProcess = *MysqlCtlProcessInstanceOptionalInit(tablet.TabletUID, tablet.MySQLPort, cluster.TmpDirectory, !cluster.ReusingVTDATAROOT)
 			proc, err := tablet.MysqlctlProcess.StartProcess()
 			if err != nil {
 				log.Errorf("error starting mysqlctl process: %v, %v", tablet.MysqlctldProcess, err)
@@ -279,6 +287,9 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames
 				cluster.VtTabletExtraArgs,
 				cluster.EnableSemiSync)
 			tablet.Alias = tablet.VttabletProcess.TabletPath
+			if cluster.ReusingVTDATAROOT {
+				tablet.VttabletProcess.ServingStatus = "SERVING"
+			}
 			shard.Vttablets = append(shard.Vttablets, tablet)
 			// Apply customizations
 			for _, customizer := range customizers {
@@ -298,9 +309,11 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames
 			}
 		}
 		for _, tablet := range shard.Vttablets {
-			if _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace.Name), keyspace.Name, false); err != nil {
-				log.Errorf("error creating database for keyspace %v: %v", keyspace.Name, err)
-				return
+			if !cluster.ReusingVTDATAROOT {
+				if _, err = tablet.VttabletProcess.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace.Name), keyspace.Name, false); err != nil {
+					log.Errorf("error creating database for keyspace %v: %v", keyspace.Name, err)
+					return
+				}
 			}
 
 			log.Infof("Starting vttablet for tablet uid %d, grpc port %d", tablet.TabletUID, tablet.GrpcPort)
@@ -358,11 +371,13 @@ func (cluster *LocalProcessCluster) SetupCluster(keyspace *Keyspace, shards []Sh
 
 	log.Infof("Starting keyspace: %v", keyspace.Name)
 
-	// Create Keyspace
-	err = cluster.VtctlProcess.CreateKeyspace(keyspace.Name)
-	if err != nil {
-		log.Error(err)
-		return
+	if !cluster.ReusingVTDATAROOT {
+		// Create Keyspace
+		err = cluster.VtctlProcess.CreateKeyspace(keyspace.Name)
+		if err != nil {
+			log.Error(err)
+			return
+		}
 	}
 
 	// Create shard
@@ -441,8 +456,18 @@ func NewCluster(cell string, hostname string) *LocalProcessCluster {
 	go cluster.CtrlCHandler()
 	cluster.OriginalVTDATAROOT = os.Getenv("VTDATAROOT")
 	cluster.CurrentVTDATAROOT = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("vtroot_%d", cluster.GetAndReservePort()))
-	_ = createDirectory(cluster.CurrentVTDATAROOT, 0700)
+	if *forceVTDATAROOT != "" {
+		cluster.CurrentVTDATAROOT = *forceVTDATAROOT
+	}
+	if _, err := os.Stat(cluster.CurrentVTDATAROOT); err == nil {
+		// path/to/whatever exists
+		cluster.ReusingVTDATAROOT = true
+	} else {
+		_ = createDirectory(cluster.CurrentVTDATAROOT, 0700)
+	}
 	_ = os.Setenv("VTDATAROOT", cluster.CurrentVTDATAROOT)
+	log.Infof("Created cluster on %s. ReusingVTDATAROOT=%v", cluster.CurrentVTDATAROOT, cluster.ReusingVTDATAROOT)
+
 	rand.Seed(time.Now().UTC().UnixNano())
 	return cluster
 }
@@ -591,7 +616,11 @@ func (cluster *LocalProcessCluster) StartVtbackup(newInitDBFile string, initalBa
 // GetAndReservePort gives port for required process
 func (cluster *LocalProcessCluster) GetAndReservePort() int {
 	if cluster.nextPortForProcess == 0 {
-		cluster.nextPortForProcess = getPort()
+		if *forcePortStart > 0 {
+			cluster.nextPortForProcess = *forcePortStart
+		} else {
+			cluster.nextPortForProcess = getPort()
+		}
 	}
 	for {
 		cluster.nextPortForProcess = cluster.nextPortForProcess + 1
@@ -634,7 +663,11 @@ func getPort() int {
 // GetAndReserveTabletUID gives tablet uid
 func (cluster *LocalProcessCluster) GetAndReserveTabletUID() int {
 	if cluster.BaseTabletUID == 0 {
-		cluster.BaseTabletUID = getRandomNumber(10000, 0)
+		if *forceBaseTabletUID > 0 {
+			cluster.BaseTabletUID = *forceBaseTabletUID
+		} else {
+			cluster.BaseTabletUID = getRandomNumber(10000, 0)
+		}
 	}
 	cluster.BaseTabletUID = cluster.BaseTabletUID + 1
 	return cluster.BaseTabletUID

go/test/endtoend/cluster/mysqlctl_process.go

@@ -161,9 +161,9 @@ func (mysqlctl *MysqlctlProcess) CleanupFiles(tabletUID int) {
 	os.RemoveAll(path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/innodb", tabletUID)))
 }
 
-// MysqlCtlProcessInstance returns a Mysqlctl handle for mysqlctl process
+// MysqlCtlProcessInstanceOptionalInit returns a Mysqlctl handle for mysqlctl process
 // configured with the given Config.
-func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) *MysqlctlProcess {
+func MysqlCtlProcessInstanceOptionalInit(tabletUID int, mySQLPort int, tmpDirectory string, initMySQL bool) *MysqlctlProcess {
 	mysqlctl := &MysqlctlProcess{
 		Name:         "mysqlctl",
 		Binary:       "mysqlctl",
@@ -172,11 +172,17 @@ func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string)
 	}
 	mysqlctl.MySQLPort = mySQLPort
 	mysqlctl.TabletUID = tabletUID
-	mysqlctl.InitMysql = true
+	mysqlctl.InitMysql = initMySQL
 	mysqlctl.SecureTransport = false
 	return mysqlctl
 }
 
+// MysqlCtlProcessInstance returns a Mysqlctl handle for mysqlctl process
+// configured with the given Config.
+func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string) *MysqlctlProcess {
+	return MysqlCtlProcessInstanceOptionalInit(tabletUID, mySQLPort, tmpDirectory, true)
+}
+
 // StartMySQL starts mysqlctl process
 func StartMySQL(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) error {
 	tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory)

go/test/endtoend/cluster/vtctlclient_process.go

@@ -49,16 +49,20 @@ func (vtctlclient *VtctlClientProcess) InitShardMaster(Keyspace string, Shard st
 
 // ApplySchemaWithOutput applies SQL schema to the keyspace
 func (vtctlclient *VtctlClientProcess) ApplySchemaWithOutput(Keyspace string, SQL string, ddlStrategy string) (result string, err error) {
-	return vtctlclient.ExecuteCommandWithOutput(
+	args := []string{
 		"ApplySchema",
 		"-sql", SQL,
-		"-ddl_strategy", ddlStrategy,
-		Keyspace)
+	}
+	if ddlStrategy != "" {
+		args = append(args, "-ddl_strategy", ddlStrategy)
+	}
+	args = append(args, Keyspace)
+	return vtctlclient.ExecuteCommandWithOutput(args...)
 }
 
 // ApplySchema applies SQL schema to the keyspace
 func (vtctlclient *VtctlClientProcess) ApplySchema(Keyspace string, SQL string) (err error) {
-	_, err = vtctlclient.ApplySchemaWithOutput(Keyspace, SQL, "")
+	_, err = vtctlclient.ApplySchemaWithOutput(Keyspace, SQL, "direct")
 	return err
 }
 
@@ -96,7 +100,7 @@ func (vtctlclient *VtctlClientProcess) OnlineDDLCancelMigration(Keyspace, uuid s
 	)
 }
 
-// OnlineDDLCancelMigration cancels a given migration uuid
+// OnlineDDLCancelAllMigrations cancels all migrations for a keyspace
 func (vtctlclient *VtctlClientProcess) OnlineDDLCancelAllMigrations(Keyspace string) (result string, err error) {
 	return vtctlclient.ExecuteCommandWithOutput(
 		"OnlineDDL",