merge master

Signed-off-by: Shlomi Noach <[email protected]>

.github/workflows/cluster_endtoend_vreplication_migrate.yml

@@ -0,0 +1,40 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Cluster (vreplication_migrate)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (vreplication_migrate)
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard vreplication_migrate

Makefile

@@ -103,7 +103,7 @@ parser:
 	make -C go/vt/sqlparser
 
 visitor:
-	go generate go/vt/sqlparser/rewriter.go
+	go run ./go/tools/asthelpergen -in ./go/vt/sqlparser -iface vitess.io/vitess/go/vt/sqlparser.SQLNode -except "*ColName"
 
 sizegen:
 	go run go/tools/sizegen/sizegen.go \

docker/k8s/Dockerfile

@@ -42,6 +42,7 @@ COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificat
 
 # Copy binaries
 COPY --from=base /vt/bin/mysqlctld /vt/bin/
+COPY --from=base /vt/bin/mysqlctl /vt/bin/
 COPY --from=base /vt/bin/vtctld /vt/bin/
 COPY --from=base /vt/bin/vtctl /vt/bin/
 COPY --from=base /vt/bin/vtctlclient /vt/bin/

docker/k8s/mysqlctl/Dockerfile

@@ -0,0 +1,42 @@
+# Copyright 2019 The Vitess Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG VT_BASE_VER=latest
+
+FROM vitess/k8s:${VT_BASE_VER} AS k8s
+
+FROM debian:buster-slim
+
+# Set up Vitess environment (just enough to run pre-built Go binaries)
+ENV VTROOT /vt
+ENV VTDATAROOT /vtdataroot
+
+# Prepare directory structure.
+RUN mkdir -p /vt/bin && \
+   mkdir -p /vt/config && mkdir -p /vtdataroot
+
+# Copy binaries
+COPY --from=k8s /vt/bin/mysqlctl /vt/bin/
+
+# Copy certs to allow https calls
+COPY --from=k8s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+# copy vitess config
+COPY --from=k8s /vt/config /vt/config
+
+# add vitess user/group and add permissions
+RUN groupadd -r --gid 2000 vitess && \
+   useradd -r -g vitess --uid 1000 vitess && \
+   chown -R vitess:vitess /vt && \
+   chown -R vitess:vitess /vtdataroot

go/test/endtoend/vreplication/cluster.go

@@ -23,37 +23,38 @@ import (
 var (
 	debug = false // set to true to always use local env vtdataroot for local debugging
 
-	originalVtdataroot string
-	vtdataroot         string
+	originalVtdataroot    string
+	vtdataroot            string
+	mainClusterConfig     *ClusterConfig
+	externalClusterConfig *ClusterConfig
 )
 
-var globalConfig = struct {
-	hostname        string
-	topoPort        int
-	vtctldPort      int
-	vtctldGrpcPort  int
-	tmpDir          string
-	vtgatePort      int
-	vtgateGrpcPort  int
-	vtgateMySQLPort int
-	tabletTypes     string
-}{"localhost", 2379, 15000, 15999, vtdataroot + "/tmp",
-	15001, 15991, 15306, "MASTER,REPLICA"}
-
-var (
-	tabletPortBase      = 15000
-	tabletGrpcPortBase  = 20000
-	tabletMysqlPortBase = 25000
-)
+// ClusterConfig defines the parameters like ports, tmpDir, tablet types which uniquely define a vitess cluster
+type ClusterConfig struct {
+	hostname            string
+	topoPort            int
+	vtctldPort          int
+	vtctldGrpcPort      int
+	vtdataroot          string
+	tmpDir              string
+	vtgatePort          int
+	vtgateGrpcPort      int
+	vtgateMySQLPort     int
+	tabletTypes         string
+	tabletPortBase      int
+	tabletGrpcPortBase  int
+	tabletMysqlPortBase int
+}
 
 // VitessCluster represents all components within the test cluster
 type VitessCluster struct {
-	Name        string
-	Cells       map[string]*Cell
-	Topo        *cluster.TopoProcess
-	Vtctld      *cluster.VtctldProcess
-	Vtctl       *cluster.VtctlProcess
-	VtctlClient *cluster.VtctlClientProcess
+	ClusterConfig *ClusterConfig
+	Name          string
+	Cells         map[string]*Cell
+	Topo          *cluster.TopoProcess
+	Vtctld        *cluster.VtctldProcess
+	Vtctl         *cluster.VtctlProcess
+	VtctlClient   *cluster.VtctlClientProcess
 }
 
 // Cell represents a Vitess cell within the test cluster
@@ -85,37 +86,66 @@ type Tablet struct {
 	DbServer *cluster.MysqlctlProcess
 }
 
-func init() {
-	originalVtdataroot = os.Getenv("VTDATAROOT")
-}
-
-func initGlobals() {
-	rand.Seed(time.Now().UTC().UnixNano())
+func setTempVtDataRoot() string {
 	dirSuffix := 100000 + rand.Intn(999999-100000) // 6 digits
 	if debug {
 		vtdataroot = originalVtdataroot
 	} else {
 		vtdataroot = path.Join(originalVtdataroot, fmt.Sprintf("vreple2e_%d", dirSuffix))
 	}
-	globalConfig.tmpDir = vtdataroot + "/tmp"
 	if _, err := os.Stat(vtdataroot); os.IsNotExist(err) {
 		os.Mkdir(vtdataroot, 0700)
 	}
 	_ = os.Setenv("VTDATAROOT", vtdataroot)
 	fmt.Printf("VTDATAROOT is %s\n", vtdataroot)
+	return vtdataroot
 }
 
-// NewVitessCluster creates an entire VitessCluster for e2e testing
-func NewVitessCluster(name string) (cluster *VitessCluster, err error) {
-	return &VitessCluster{Name: name, Cells: make(map[string]*Cell)}, nil
+func getClusterConfig(idx int, dataRootDir string) *ClusterConfig {
+	basePort := 15000
+	etcdPort := 2379
+
+	basePort += idx * 10000
+	etcdPort += idx * 10000
+	if _, err := os.Stat(dataRootDir); os.IsNotExist(err) {
+		os.Mkdir(dataRootDir, 0700)
+	}
+
+	return &ClusterConfig{
+		hostname:            "localhost",
+		topoPort:            etcdPort,
+		vtctldPort:          basePort,
+		vtctldGrpcPort:      basePort + 999,
+		tmpDir:              dataRootDir + "/tmp",
+		vtgatePort:          basePort + 1,
+		vtgateGrpcPort:      basePort + 991,
+		vtgateMySQLPort:     basePort + 306,
+		tabletTypes:         "master",
+		vtdataroot:          dataRootDir,
+		tabletPortBase:      basePort + 1000,
+		tabletGrpcPortBase:  basePort + 1991,
+		tabletMysqlPortBase: basePort + 1306,
+	}
+}
+
+func init() {
+	rand.Seed(time.Now().UTC().UnixNano())
+	originalVtdataroot = os.Getenv("VTDATAROOT")
+	var mainVtDataRoot string
+	if debug {
+		mainVtDataRoot = originalVtdataroot
+	} else {
+		mainVtDataRoot = setTempVtDataRoot()
+	}
+	mainClusterConfig = getClusterConfig(0, mainVtDataRoot)
+	externalClusterConfig = getClusterConfig(1, mainVtDataRoot+"/ext")
 }
 
-// InitCluster creates the global processes needed for a cluster
-func InitCluster(t *testing.T, cellNames []string) *VitessCluster {
-	initGlobals()
-	vc, _ := NewVitessCluster("Vdemo")
+// NewVitessCluster starts a basic cluster with vtgate, vtctld and the topo
+func NewVitessCluster(t *testing.T, name string, cellNames []string, clusterConfig *ClusterConfig) *VitessCluster {
+	vc := &VitessCluster{Name: name, Cells: make(map[string]*Cell), ClusterConfig: clusterConfig}
 	require.NotNil(t, vc)
-	topo := cluster.TopoProcessInstance(globalConfig.topoPort, globalConfig.topoPort*10, globalConfig.hostname, "etcd2", "global")
+	topo := cluster.TopoProcessInstance(vc.ClusterConfig.topoPort, vc.ClusterConfig.topoPort+1, vc.ClusterConfig.hostname, "etcd2", "global")
 
 	require.NotNil(t, topo)
 	require.Nil(t, topo.Setup("etcd2", nil))
@@ -125,14 +155,14 @@ func InitCluster(t *testing.T, cellNames []string) *VitessCluster {
 		topo.ManageTopoDir("mkdir", "/vitess/"+cellName)
 	}
 
-	vtctld := cluster.VtctldProcessInstance(globalConfig.vtctldPort, globalConfig.vtctldGrpcPort,
-		globalConfig.topoPort, globalConfig.hostname, globalConfig.tmpDir)
+	vtctld := cluster.VtctldProcessInstance(vc.ClusterConfig.vtctldPort, vc.ClusterConfig.vtctldGrpcPort,
+		vc.ClusterConfig.topoPort, vc.ClusterConfig.hostname, vc.ClusterConfig.tmpDir)
 	vc.Vtctld = vtctld
 	require.NotNil(t, vc.Vtctld)
 	// use first cell as `-cell`
 	vc.Vtctld.Setup(cellNames[0])
 
-	vc.Vtctl = cluster.VtctlProcessInstance(globalConfig.topoPort, globalConfig.hostname)
+	vc.Vtctl = cluster.VtctlProcessInstance(vc.ClusterConfig.topoPort, vc.ClusterConfig.hostname)
 	require.NotNil(t, vc.Vtctl)
 	for _, cellName := range cellNames {
 		vc.Vtctl.AddCellInfo(cellName)
@@ -141,7 +171,7 @@ func InitCluster(t *testing.T, cellNames []string) *VitessCluster {
 		require.NotNil(t, cell)
 	}
 
-	vc.VtctlClient = cluster.VtctlClientProcessInstance(globalConfig.hostname, vc.Vtctld.GrpcPort, globalConfig.tmpDir)
+	vc.VtctlClient = cluster.VtctlClientProcessInstance(vc.ClusterConfig.hostname, vc.Vtctld.GrpcPort, vc.ClusterConfig.tmpDir)
 	require.NotNil(t, vc.VtctlClient)
 
 	return vc
@@ -194,17 +224,17 @@ func (vc *VitessCluster) AddTablet(t *testing.T, cell *Cell, keyspace *Keyspace,
 	tablet := &Tablet{}
 
 	vttablet := cluster.VttabletProcessInstance(
-		tabletPortBase+tabletID,
-		tabletGrpcPortBase+tabletID,
+		vc.ClusterConfig.tabletPortBase+tabletID,
+		vc.ClusterConfig.tabletGrpcPortBase+tabletID,
 		tabletID,
 		cell.Name,
 		shard.Name,
 		keyspace.Name,
-		globalConfig.vtctldPort,
+		vc.ClusterConfig.vtctldPort,
 		tabletType,
 		vc.Topo.Port,
-		globalConfig.hostname,
-		globalConfig.tmpDir,
+		vc.ClusterConfig.hostname,
+		vc.ClusterConfig.tmpDir,
 		[]string{
 			"-queryserver-config-schema-reload-time", "5",
 			"-enable-lag-throttler",
@@ -216,7 +246,7 @@ func (vc *VitessCluster) AddTablet(t *testing.T, cell *Cell, keyspace *Keyspace,
 	require.NotNil(t, vttablet)
 	vttablet.SupportsBackup = false
 
-	tablet.DbServer = cluster.MysqlCtlProcessInstance(tabletID, tabletMysqlPortBase+tabletID, globalConfig.tmpDir)
+	tablet.DbServer = cluster.MysqlCtlProcessInstance(tabletID, vc.ClusterConfig.tabletMysqlPortBase+tabletID, vc.ClusterConfig.tmpDir)
 	require.NotNil(t, tablet.DbServer)
 	tablet.DbServer.InitMysql = true
 	proc, err := tablet.DbServer.StartProcess()
@@ -331,15 +361,15 @@ func (vc *VitessCluster) DeleteShard(t *testing.T, cellName string, ksName strin
 // StartVtgate starts a vtgate process
 func (vc *VitessCluster) StartVtgate(t *testing.T, cell *Cell, cellsToWatch string) {
 	vtgate := cluster.VtgateProcessInstance(
-		globalConfig.vtgatePort,
-		globalConfig.vtgateGrpcPort,
-		globalConfig.vtgateMySQLPort,
+		vc.ClusterConfig.vtgatePort,
+		vc.ClusterConfig.vtgateGrpcPort,
+		vc.ClusterConfig.vtgateMySQLPort,
 		cell.Name,
 		cellsToWatch,
-		globalConfig.hostname,
-		globalConfig.tabletTypes,
-		globalConfig.topoPort,
-		globalConfig.tmpDir,
+		vc.ClusterConfig.hostname,
+		vc.ClusterConfig.tabletTypes,
+		vc.ClusterConfig.topoPort,
+		vc.ClusterConfig.tmpDir,
 		[]string{"-tablet_refresh_interval", "10ms"})
 	require.NotNil(t, vtgate)
 	if err := vtgate.Setup(); err != nil {

go/test/endtoend/vreplication/config.go

@@ -287,5 +287,18 @@ create table customer_seq2(id int, next_id bigint, cache bigint, primary key(id)
     "create_ddl": "create table rollup(rollupname varchar(100), kount int, primary key (rollupname))"
   }]
 }
+`
+	initialExternalSchema = `
+create table review(rid int, pid int, review varbinary(128), primary key(rid));
+create table rating(gid int, pid int, rating int, primary key(gid));
+`
+
+	initialExternalVSchema = `
+{
+  "tables": {
+	"review": {},
+	"rating": {}
+  }
+}
 `
 )

go/test/endtoend/vreplication/helper.go

@@ -33,9 +33,9 @@ func execQuery(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
 	return qr
 }
 
-func getConnection(t *testing.T, port int) *mysql.Conn {
+func getConnection(t *testing.T, hostname string, port int) *mysql.Conn {
 	vtParams := mysql.ConnParams{
-		Host:  globalConfig.hostname,
+		Host:  hostname,
 		Port:  port,
 		Uname: "vt_dba",
 	}