Alter tests to reuse cluster components

go/test/endtoend/backup/transform/backup_transform_utils.go

@@ -250,7 +250,7 @@ func TestBackupTransformImpl(t *testing.T) {
 	replica2.VttabletProcess.ServingStatus = ""
 	err = replica2.VttabletProcess.Setup()
 	require.Nil(t, err)
-	err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second)
+	err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second)
 	require.Nil(t, err)
 	defer replica2.VttabletProcess.TearDown()
 

go/test/endtoend/backup/vtctlbackup/backup_utils.go

@@ -295,7 +295,7 @@ func masterBackup(t *testing.T) {
 	require.Nil(t, err)
 
 	restoreWaitForBackup(t, "replica")
-	err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second)
+	err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second)
 	require.Nil(t, err)
 
 	cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2)
@@ -327,7 +327,7 @@ func masterReplicaSameBackup(t *testing.T) {
 
 	// now bring up the other replica, letting it restore from backup.
 	restoreWaitForBackup(t, "replica")
-	err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second)
+	err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second)
 	require.Nil(t, err)
 
 	// check the new replica has the data
@@ -563,7 +563,7 @@ func vtctlBackup(t *testing.T, tabletType string) {
 	_, err = master.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true)
 	require.Nil(t, err)
 
-	err = replica2.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 25*time.Second)
+	err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second)
 	require.Nil(t, err)
 	cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2)
 
@@ -626,7 +626,7 @@ func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string)
 	err := tablet.VttabletProcess.Setup()
 	require.Nil(t, err)
 	if status != "" {
-		err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{status}, 25*time.Second)
+		err = tablet.VttabletProcess.WaitForTabletStatusesForTimeout([]string{status}, 25*time.Second)
 		require.Nil(t, err)
 	}
 

go/test/endtoend/cluster/vtctlclient_process.go

@@ -156,8 +156,6 @@ func (vtctlclient *VtctlClientProcess) ExecuteCommand(args ...string) (err error
 	if output != "" {
 		if err != nil {
 			log.Errorf("Output:\n%v", output)
-		} else {
-			log.Infof("Output:\n%v", output)
 		}
 	}
 	return err

go/test/endtoend/cluster/vtorc_process.go

@@ -18,6 +18,7 @@ limitations under the License.
 package cluster
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"path"
@@ -52,6 +53,7 @@ func (orc *VtorcProcess) Setup() (err error) {
 		"-topo_global_server_address", orc.TopoGlobalAddress,
 		"-topo_global_root", orc.TopoGlobalRoot,
 		"-config", orc.Config,
+		"-orc_web_dir", path.Join(os.Getenv("VTROOT"), "web", "orchestrator"),
 	)
 	if *isCoverage {
 		orc.proc.Args = append(orc.proc.Args, "-test.coverprofile="+getCoveragePath("orc.out"))
@@ -60,7 +62,7 @@ func (orc *VtorcProcess) Setup() (err error) {
 	orc.proc.Args = append(orc.proc.Args, orc.ExtraArgs...)
 	orc.proc.Args = append(orc.proc.Args, "-alsologtostderr", "http")
 
-	errFile, _ := os.Create(path.Join(orc.LogDir, "orc-stderr.txt"))
+	errFile, _ := os.Create(path.Join(orc.LogDir, fmt.Sprintf("orc-stderr-%d.txt", time.Now().UnixNano())))
 	orc.proc.Stderr = errFile
 
 	orc.proc.Env = append(orc.proc.Env, os.Environ()...)
@@ -95,7 +97,7 @@ func (orc *VtorcProcess) TearDown() error {
 		orc.proc = nil
 		return nil
 
-	case <-time.After(10 * time.Second):
+	case <-time.After(30 * time.Second):
 		_ = orc.proc.Process.Kill()
 		orc.proc = nil
 		return <-orc.exit

go/test/endtoend/cluster/vttablet_process.go

@@ -140,7 +140,7 @@ func (vttablet *VttabletProcess) Setup() (err error) {
 	}()
 
 	if vttablet.ServingStatus != "" {
-		if err = vttablet.WaitForTabletType(vttablet.ServingStatus); err != nil {
+		if err = vttablet.WaitForTabletStatus(vttablet.ServingStatus); err != nil {
 			errFileContent, _ := ioutil.ReadFile(fname)
 			if errFileContent != nil {
 				log.Infof("vttablet error:\n%s\n", string(errFileContent))
@@ -208,23 +208,37 @@ func (vttablet *VttabletProcess) GetTabletStatus() string {
 	return ""
 }
 
-// WaitForTabletType waits for 10 second till expected type reached
-func (vttablet *VttabletProcess) WaitForTabletType(expectedType string) error {
-	return vttablet.WaitForTabletTypesForTimeout([]string{expectedType}, 10*time.Second)
+// GetTabletType returns the tablet type as seen in /debug/vars TabletType
+func (vttablet *VttabletProcess) GetTabletType() string {
+	resultMap := vttablet.GetVars()
+	if resultMap != nil {
+		return reflect.ValueOf(resultMap["TabletType"]).String()
+	}
+	return ""
+}
+
+// WaitForTabletStatus waits for 10 second till expected status is reached
+func (vttablet *VttabletProcess) WaitForTabletStatus(expectedStatus string) error {
+	return vttablet.WaitForTabletStatusesForTimeout([]string{expectedStatus}, 10*time.Second)
 }
 
-// WaitForTabletTypes waits for 10 second till expected type reached
+// WaitForTabletStatuses waits for 10 second till one of expected statuses is reached
+func (vttablet *VttabletProcess) WaitForTabletStatuses(expectedStatuses []string) error {
+	return vttablet.WaitForTabletStatusesForTimeout(expectedStatuses, 10*time.Second)
+}
+
+// WaitForTabletTypes waits for 10 second till one of expected statuses is reached
 func (vttablet *VttabletProcess) WaitForTabletTypes(expectedTypes []string) error {
 	return vttablet.WaitForTabletTypesForTimeout(expectedTypes, 10*time.Second)
 }
 
-// WaitForTabletTypesForTimeout waits till the tablet reaches to any of the provided status
-func (vttablet *VttabletProcess) WaitForTabletTypesForTimeout(expectedTypes []string, timeout time.Duration) error {
-	timeToWait := time.Now().Add(timeout)
+// WaitForTabletStatusesForTimeout waits till the tablet reaches to any of the provided statuses
+func (vttablet *VttabletProcess) WaitForTabletStatusesForTimeout(expectedStatuses []string, timeout time.Duration) error {
+	waitUntil := time.Now().Add(timeout)
 	var status string
-	for time.Now().Before(timeToWait) {
+	for time.Now().Before(waitUntil) {
 		status = vttablet.GetTabletStatus()
-		if contains(expectedTypes, status) {
+		if contains(expectedStatuses, status) {
 			return nil
 		}
 		select {
@@ -235,7 +249,27 @@ func (vttablet *VttabletProcess) WaitForTabletTypesForTimeout(expectedTypes []st
 		}
 	}
 	return fmt.Errorf("Vttablet %s, current status = %s, expected status [%s] not reached, details: %v",
-		vttablet.TabletPath, status, strings.Join(expectedTypes, ","), vttablet.GetStatusDetails())
+		vttablet.TabletPath, status, strings.Join(expectedStatuses, ","), vttablet.GetStatusDetails())
+}
+
+// WaitForTabletTypesForTimeout waits till the tablet reaches to any of the provided types
+func (vttablet *VttabletProcess) WaitForTabletTypesForTimeout(expectedTypes []string, timeout time.Duration) error {
+	waitUntil := time.Now().Add(timeout)
+	var tabletType string
+	for time.Now().Before(waitUntil) {
+		tabletType = vttablet.GetTabletType()
+		if contains(expectedTypes, tabletType) {
+			return nil
+		}
+		select {
+		case err := <-vttablet.exit:
+			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err)
+		default:
+			time.Sleep(300 * time.Millisecond)
+		}
+	}
+	return fmt.Errorf("Vttablet %s, current type = %s, expected type [%s] not reached, status details: %v",
+		vttablet.TabletPath, tabletType, strings.Join(expectedTypes, ","), vttablet.GetStatusDetails())
 }
 
 func contains(arr []string, str string) bool {
@@ -472,6 +506,11 @@ func (vttablet *VttabletProcess) BulkLoad(t testing.TB, db, table string, bulkIn
 		bufFinish.Sub(bufStart), end.Sub(bufFinish), end.Sub(bufStart))
 }
 
+// IsShutdown returns whether a vttablet is shutdown or not
+func (vttablet *VttabletProcess) IsShutdown() bool {
+	return vttablet.proc == nil
+}
+
 // VttabletProcessInstance returns a VttabletProcess handle for vttablet process
 // configured with the given Config.
 // The process must be manually started by calling setup()

go/test/endtoend/recovery/pitr/shardedpitr_test.go

@@ -538,5 +538,5 @@ func launchRecoveryTablet(t *testing.T, tablet *cluster.Vttablet, binlogServer *
 	err = tablet.VttabletProcess.Setup()
 	require.NoError(t, err)
 
-	tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 20*time.Second)
+	tablet.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 20*time.Second)
 }

go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go

@@ -522,7 +522,7 @@ func tlsLaunchRecoveryTablet(t *testing.T, tablet *cluster.Vttablet, tabletForBi
 	err = tablet.VttabletProcess.Setup()
 	require.NoError(t, err)
 
-	tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 20*time.Second)
+	tablet.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 20*time.Second)
 }
 
 func getCNFromCertPEM(filename string) string {

go/test/endtoend/recovery/recovery_util.go

@@ -83,7 +83,7 @@ func RestoreTablet(t *testing.T, localCluster *cluster.LocalProcessCluster, tabl
 	err = tablet.VttabletProcess.Setup()
 	require.Nil(t, err)
 
-	err = tablet.VttabletProcess.WaitForTabletTypesForTimeout([]string{"SERVING"}, 20*time.Second)
+	err = tablet.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 20*time.Second)
 	require.Nil(t, err)
 }
 

go/test/endtoend/recovery/shardedrecovery/sharded_recovery_test.go

@@ -149,7 +149,7 @@ func TestUnShardedRecoveryAfterSharding(t *testing.T) {
 	shardedTablets := []*cluster.Vttablet{shard0Master, shard0Replica, shard0RdOnly, shard1Master, shard1Replica, shard1RdOnly}
 
 	for _, tablet := range shardedTablets {
-		_ = tablet.VttabletProcess.WaitForTabletType("SERVING")
+		_ = tablet.VttabletProcess.WaitForTabletStatus("SERVING")
 		require.NoError(t, err)
 	}
 
@@ -288,7 +288,7 @@ func TestShardedRecovery(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, tablet := range shardedTablets {
-		_ = tablet.VttabletProcess.WaitForTabletType("SERVING")
+		_ = tablet.VttabletProcess.WaitForTabletStatus("SERVING")
 		require.NoError(t, err)
 	}
 

go/test/endtoend/reparent/utils_test.go

@@ -163,7 +163,7 @@ func setupShard(ctx context.Context, t *testing.T, shardName string, tablets []*
 	}
 
 	for _, tablet := range tablets {
-		err := tablet.VttabletProcess.WaitForTabletTypes([]string{"SERVING", "NOT_SERVING"})
+		err := tablet.VttabletProcess.WaitForTabletStatuses([]string{"SERVING", "NOT_SERVING"})
 		require.NoError(t, err)
 	}
 

go/test/endtoend/sharding/initialsharding/sharding_util.go

@@ -271,15 +271,15 @@ func TestInitialSharding(t *testing.T, keyspace *cluster.Keyspace, keyType query
 		err = ClusterInstance.VtctlclientProcess.InitShardMaster(keyspace.Name, shard1.Name, cell, shard1MasterTablet.TabletUID)
 		require.NoError(t, err)
 	} else {
-		err = shard1.Replica().VttabletProcess.WaitForTabletType("SERVING")
+		err = shard1.Replica().VttabletProcess.WaitForTabletStatus("SERVING")
 		require.NoError(t, err)
 		_, err = ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("TabletExternallyReparented", shard1MasterTablet.Alias)
 		require.NoError(t, err)
 	}
 
-	err = shard1.Replica().VttabletProcess.WaitForTabletType("SERVING")
+	err = shard1.Replica().VttabletProcess.WaitForTabletStatus("SERVING")
 	require.NoError(t, err)
-	err = shard1.Rdonly().VttabletProcess.WaitForTabletType("SERVING")
+	err = shard1.Rdonly().VttabletProcess.WaitForTabletStatus("SERVING")
 	require.NoError(t, err)
 	for _, vttablet := range shard1.Vttablets {
 		assert.Equal(t, vttablet.VttabletProcess.GetTabletStatus(), "SERVING")
@@ -352,8 +352,8 @@ func TestInitialSharding(t *testing.T, keyspace *cluster.Keyspace, keyType query
 		_ = ClusterInstance.VtctlclientProcess.ApplyVSchema(keyspaceName, fmt.Sprintf(vSchema, tableName, "id"))
 
 		for _, shard := range []cluster.Shard{shard21, shard22} {
-			_ = shard.Replica().VttabletProcess.WaitForTabletType("SERVING")
-			_ = shard.Rdonly().VttabletProcess.WaitForTabletType("SERVING")
+			_ = shard.Replica().VttabletProcess.WaitForTabletStatus("SERVING")
+			_ = shard.Rdonly().VttabletProcess.WaitForTabletStatus("SERVING")
 		}
 
 		for _, shard := range []cluster.Shard{shard21, shard22} {
@@ -526,8 +526,8 @@ func TestInitialSharding(t *testing.T, keyspace *cluster.Keyspace, keyType query
 	expectedPartitions[topodata.TabletType_RDONLY] = []string{shard21.Name, shard22.Name}
 	checkSrvKeyspaceForSharding(t, keyspaceName, expectedPartitions)
 
-	_ = shard21.Rdonly().VttabletProcess.WaitForTabletType("SERVING")
-	_ = shard22.Rdonly().VttabletProcess.WaitForTabletType("SERVING")
+	_ = shard21.Rdonly().VttabletProcess.WaitForTabletStatus("SERVING")
+	_ = shard22.Rdonly().VttabletProcess.WaitForTabletStatus("SERVING")
 
 	_ = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspaceName, shard21.Name), 1)
 	_ = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspaceName, shard22.Name), 1)

go/test/endtoend/sharding/mergesharding/mergesharding_base.go

@@ -214,13 +214,13 @@ func TestMergesharding(t *testing.T, useVarbinaryShardingKeyType bool) {
 	require.NoError(t, err)
 
 	// Wait for tablets to come in Service state
-	err = shard0Master.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard0Master.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.NoError(t, err)
-	err = shard1Master.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard1Master.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.NoError(t, err)
-	err = shard2Master.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard2Master.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.NoError(t, err)
-	err = shard3Master.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard3Master.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.NoError(t, err)
 
 	// keyspace/shard name fields
@@ -348,7 +348,7 @@ func TestMergesharding(t *testing.T, useVarbinaryShardingKeyType bool) {
 	// The tested behavior is a safeguard to prevent that somebody can
 	// accidentally modify data on the destination masters while they are not
 	// migrated yet and the source shards are still the source of truth.
-	err = shard3Master.VttabletProcess.WaitForTabletType("NOT_SERVING")
+	err = shard3Master.VttabletProcess.WaitForTabletStatus("NOT_SERVING")
 	require.NoError(t, err)
 
 	// check that binlog server exported the stats vars

go/test/endtoend/sharding/resharding/resharding_base.go

@@ -305,13 +305,13 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) {
 	require.Nil(t, err)
 
 	// Wait for tablets to come in Service state
-	err = shard0Master.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard0Master.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.Nil(t, err)
-	err = shard1Master.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard1Master.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.Nil(t, err)
-	err = shard2Master.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard2Master.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.Nil(t, err)
-	err = shard3Master.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard3Master.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.Nil(t, err)
 
 	// keyspace/shard name fields
@@ -375,7 +375,7 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) {
 	err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", shard1Replica2.Alias, "spare")
 	require.Nil(t, err)
 
-	err = shard1Replica2.VttabletProcess.WaitForTabletType("NOT_SERVING")
+	err = shard1Replica2.VttabletProcess.WaitForTabletStatus("NOT_SERVING")
 	require.Nil(t, err)
 
 	// we need to create the schema, and the worker will do data copying
@@ -522,9 +522,9 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) {
 	// The tested behavior is a safeguard to prevent that somebody can
 	// accidentally modify data on the destination masters while they are not
 	// migrated yet and the source shards are still the source of truth.
-	err = shard2Master.VttabletProcess.WaitForTabletType("NOT_SERVING")
+	err = shard2Master.VttabletProcess.WaitForTabletStatus("NOT_SERVING")
 	require.Nil(t, err)
-	err = shard3Master.VttabletProcess.WaitForTabletType("NOT_SERVING")
+	err = shard3Master.VttabletProcess.WaitForTabletStatus("NOT_SERVING")
 	require.Nil(t, err)
 
 	// check that binlog server exported the stats vars
@@ -596,9 +596,9 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) {
 	err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", shard1Replica1.Alias, "spare")
 	require.Nil(t, err)
 
-	err = shard1Replica2.VttabletProcess.WaitForTabletType("SERVING")
+	err = shard1Replica2.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.Nil(t, err)
-	err = shard1Replica1.VttabletProcess.WaitForTabletType("NOT_SERVING")
+	err = shard1Replica1.VttabletProcess.WaitForTabletStatus("NOT_SERVING")
 	require.Nil(t, err)
 
 	err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", shard1Replica2.Alias)

go/test/endtoend/sharding/verticalsplit/vertical_split_test.go

@@ -141,11 +141,11 @@ func TestVerticalSplit(t *testing.T) {
 	destinationMasterTablet.Type = "master"
 
 	for _, tablet := range []cluster.Vttablet{sourceReplicaTablet, destinationReplicaTablet} {
-		_ = tablet.VttabletProcess.WaitForTabletType("SERVING")
+		_ = tablet.VttabletProcess.WaitForTabletStatus("SERVING")
 		require.NoError(t, err)
 	}
 	for _, tablet := range []cluster.Vttablet{sourceRdOnlyTablet1, sourceRdOnlyTablet2, destinationRdOnlyTablet1, destinationRdOnlyTablet2} {
-		_ = tablet.VttabletProcess.WaitForTabletType("SERVING")
+		_ = tablet.VttabletProcess.WaitForTabletStatus("SERVING")
 		require.NoError(t, err)
 	}
 

go/test/endtoend/tabletmanager/master/tablet_master_test.go

@@ -169,7 +169,7 @@ func TestMasterRestartSetsTERTimestamp(t *testing.T) {
 	err := clusterInstance.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, replicaTablet.TabletUID)
 	require.Nil(t, err)
 
-	err = replicaTablet.VttabletProcess.WaitForTabletType("SERVING")
+	err = replicaTablet.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.Nil(t, err)
 
 	// Capture the current TER.
@@ -224,7 +224,7 @@ func TestMasterRestartSetsTERTimestamp(t *testing.T) {
 	// Reset master
 	err = clusterInstance.VtctlclientProcess.InitShardMaster(keyspaceName, shardName, cell, masterTablet.TabletUID)
 	require.Nil(t, err)
-	err = masterTablet.VttabletProcess.WaitForTabletType("SERVING")
+	err = masterTablet.VttabletProcess.WaitForTabletStatus("SERVING")
 	require.Nil(t, err)
 
 }