Merge remote-tracking branch 'upstream/master' into morgo-matrix-unit

Signed-off-by: Morgan Tocker <[email protected]>

.github/workflows/check_make_parser.yml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Build
+    name: Check Make Parser
     runs-on: ubuntu-latest
     steps:
 

.github/workflows/cluster_endtoend.yml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Build
+    name: Cluster End-to-End Test
     runs-on: ubuntu-latest
     steps:
 

.github/workflows/e2e_race.yml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Build
+    name: End-to-End Test (Race)
     runs-on: ubuntu-latest
     steps:
 

.github/workflows/endtoend.yml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Build
+    name: End-to-End Test
     runs-on: ubuntu-latest
     steps:
 

.github/workflows/local_example.yml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Build
+    name: Local Example Test
     runs-on: ubuntu-latest
     steps:
 

.github/workflows/unit.yml

@@ -37,11 +37,13 @@ jobs:
           sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
         fi
 
-        sudo apt-get install -y make unzip g++ etcd curl git wget ant openjdk-8-jdk
+        sudo apt-get install -y make unzip g++ curl git wget ant openjdk-8-jdk
         sudo service mysql stop
-        sudo service etcd stop
         sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        mkdir -p dist bin
+        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
+        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
         go mod download
 
     - name: Run make tools

.github/workflows/unit_race.yml

@@ -3,7 +3,7 @@ on: [push]
 jobs:
 
   build:
-    name: Build
+    name: Unit Test (Race)
     runs-on: ubuntu-latest
     steps:
 

docker/bootstrap/Dockerfile.mariadb

@@ -22,5 +22,6 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
+ENV MYSQL_FLAVOR MariaDB
 USER vitess
 RUN ./bootstrap.sh

docker/bootstrap/Dockerfile.mariadb103

@@ -11,5 +11,6 @@ RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
+ENV MYSQL_FLAVOR MariaDB103
 USER vitess
 RUN ./bootstrap.sh

docker/bootstrap/Dockerfile.mysql80

@@ -17,5 +17,6 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.poo
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
+ENV MYSQL_FLAVOR MySQL80
 USER vitess
-RUN ./bootstrap.sh
+RUN ./bootstrap.sh

docker/bootstrap/Dockerfile.percona80

@@ -32,5 +32,6 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
+ENV MYSQL_FLAVOR MySQL80
 USER vitess
 RUN ./bootstrap.sh

docker/lite/Dockerfile.alpine

@@ -25,6 +25,7 @@ RUN echo '@edge http://nl.alpinelinux.org/alpine/edge/main' >> /etc/apk/reposito
 ENV VTROOT /vt/src/vitess.io/vitess
 ENV VTDATAROOT /vt/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
+ENV MYSQL_FLAVOR MariaDB103
 
 # Create vitess user
 RUN addgroup -S vitess && adduser -S -G vitess vitess && mkdir -p /vt

docker/lite/Dockerfile.mariadb

@@ -36,6 +36,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
 ENV VTROOT /vt/src/vitess.io/vitess
 ENV VTDATAROOT /vt/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
+ENV MYSQL_FLAVOR MariaDB
 
 # Copy binaries (placed by build.sh)
 COPY --from=staging /vt/ /vt/

docker/lite/Dockerfile.mariadb103

@@ -35,6 +35,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
 ENV VTROOT /vt/src/vitess.io/vitess
 ENV VTDATAROOT /vt/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
+ENV MYSQL_FLAVOR MariaDB103
 
 # Copy binaries (placed by build.sh)
 COPY --from=staging /vt/ /vt/

docker/lite/Dockerfile.mysql80

@@ -39,6 +39,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
 ENV VTROOT /vt/src/vitess.io/vitess
 ENV VTDATAROOT /vt/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
+ENV MYSQL_FLAVOR MySQL80
 
 # Copy binaries (placed by build.sh)
 COPY --from=staging /vt/ /vt/

docker/lite/Dockerfile.percona80

@@ -44,6 +44,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
 ENV VTROOT /vt/src/vitess.io/vitess
 ENV VTDATAROOT /vt/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
+ENV MYSQL_FLAVOR MySQL80
 
 # Copy binaries (placed by build.sh)
 COPY --from=staging /vt/ /vt/

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/golang/mock v1.3.1
 	github.com/golang/protobuf v1.3.2
 	github.com/golang/snappy v0.0.0-20170215233205-553a64147049
+	github.com/google/btree v1.0.0 // indirect
 	github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect
 	github.com/gorilla/websocket v0.0.0-20160912153041-2d1e4548da23
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
@@ -49,6 +50,8 @@ require (
 	github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1
 	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984
 	github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02
 	github.com/opentracing/opentracing-go v1.1.0

go/vt/tlstest/tlstest.go

@@ -147,3 +147,51 @@ func CreateSignedCert(root, parent, serial, name, commonName string) {
 		"-extfile", config,
 		"-out", cert)
 }
+
+type ClientServerKeyPairs struct {
+	ServerCert string
+	ServerKey  string
+	ServerCA   string
+	ServerName string
+	ClientCert string
+	ClientKey  string
+	ClientCA   string
+}
+
+var serialCounter = 0
+
+func CreateClientServerCertPairs(root string) ClientServerKeyPairs {
+	// Create the certs and configs.
+	CreateCA(root)
+
+	serverSerial := fmt.Sprintf("%03d", serialCounter*2+1)
+	clientSerial := fmt.Sprintf("%03d", serialCounter*2+2)
+
+	serialCounter = serialCounter + 1
+
+	serverName := fmt.Sprintf("server-%s", serverSerial)
+	serverCACommonName := fmt.Sprintf("Server %s CA", serverSerial)
+	serverCertName := fmt.Sprintf("server-instance-%s", serverSerial)
+	serverCertCommonName := fmt.Sprintf("server%s.example.com", serverSerial)
+
+	clientName := fmt.Sprintf("clients-%s", serverSerial)
+	clientCACommonName := fmt.Sprintf("Clients %s CA", serverSerial)
+	clientCertName := fmt.Sprintf("client-instance-%s", serverSerial)
+	clientCertCommonName := fmt.Sprintf("Client Instance %s", serverSerial)
+
+	CreateSignedCert(root, CA, serverSerial, serverName, serverCACommonName)
+	CreateSignedCert(root, serverName, serverSerial, serverCertName, serverCertCommonName)
+
+	CreateSignedCert(root, CA, clientSerial, clientName, clientCACommonName)
+	CreateSignedCert(root, clientName, serverSerial, clientCertName, clientCertCommonName)
+
+	return ClientServerKeyPairs{
+		ServerCert: path.Join(root, fmt.Sprintf("%s-cert.pem", serverCertName)),
+		ServerKey:  path.Join(root, fmt.Sprintf("%s-key.pem", serverCertName)),
+		ServerCA:   path.Join(root, fmt.Sprintf("%s-cert.pem", serverName)),
+		ClientCert: path.Join(root, fmt.Sprintf("%s-cert.pem", clientCertName)),
+		ClientKey:  path.Join(root, fmt.Sprintf("%s-key.pem", clientCertName)),
+		ClientCA:   path.Join(root, fmt.Sprintf("%s-cert.pem", clientName)),
+		ServerName: serverCertCommonName,
+	}
+}

go/vt/tlstest/tlstest_test.go

@@ -24,7 +24,6 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
-	"path"
 	"strings"
 	"sync"
 	"testing"
@@ -47,20 +46,20 @@ func TestClientServer(t *testing.T) {
 	}
 	defer os.RemoveAll(root)
 
-	clientServerKeyPairs := createClientServerCertPairs(root)
+	clientServerKeyPairs := CreateClientServerCertPairs(root)
 
 	serverConfig, err := vttls.ServerConfig(
-		clientServerKeyPairs.serverCert,
-		clientServerKeyPairs.serverKey,
-		clientServerKeyPairs.clientCA)
+		clientServerKeyPairs.ServerCert,
+		clientServerKeyPairs.ServerKey,
+		clientServerKeyPairs.ClientCA)
 	if err != nil {
 		t.Fatalf("TLSServerConfig failed: %v", err)
 	}
 	clientConfig, err := vttls.ClientConfig(
-		clientServerKeyPairs.clientCert,
-		clientServerKeyPairs.clientKey,
-		clientServerKeyPairs.serverCA,
-		clientServerKeyPairs.serverName)
+		clientServerKeyPairs.ClientCert,
+		clientServerKeyPairs.ClientKey,
+		clientServerKeyPairs.ServerCA,
+		clientServerKeyPairs.ServerName)
 	if err != nil {
 		t.Fatalf("TLSClientConfig failed: %v", err)
 	}
@@ -117,10 +116,10 @@ func TestClientServer(t *testing.T) {
 	//
 
 	badClientConfig, err := vttls.ClientConfig(
-		clientServerKeyPairs.serverCert,
-		clientServerKeyPairs.serverKey,
-		clientServerKeyPairs.serverCA,
-		clientServerKeyPairs.serverName)
+		clientServerKeyPairs.ServerCert,
+		clientServerKeyPairs.ServerKey,
+		clientServerKeyPairs.ServerCA,
+		clientServerKeyPairs.ServerName)
 	if err != nil {
 		t.Fatalf("TLSClientConfig failed: %v", err)
 	}
@@ -165,69 +164,19 @@ func TestClientServer(t *testing.T) {
 	}
 }
 
-var serialCounter = 0
-
-type clientServerKeyPairs struct {
-	serverCert string
-	serverKey  string
-	serverCA   string
-	serverName string
-	clientCert string
-	clientKey  string
-	clientCA   string
-}
-
-func createClientServerCertPairs(root string) clientServerKeyPairs {
-
-	// Create the certs and configs.
-	CreateCA(root)
-
-	serverSerial := fmt.Sprintf("%03d", serialCounter*2+1)
-	clientSerial := fmt.Sprintf("%03d", serialCounter*2+2)
-
-	serialCounter = serialCounter + 1
-
-	serverName := fmt.Sprintf("server-%s", serverSerial)
-	serverCACommonName := fmt.Sprintf("Server %s CA", serverSerial)
-	serverCertName := fmt.Sprintf("server-instance-%s", serverSerial)
-	serverCertCommonName := fmt.Sprintf("server%s.example.com", serverSerial)
-
-	clientName := fmt.Sprintf("clients-%s", serverSerial)
-	clientCACommonName := fmt.Sprintf("Clients %s CA", serverSerial)
-	clientCertName := fmt.Sprintf("client-instance-%s", serverSerial)
-	clientCertCommonName := fmt.Sprintf("Client Instance %s", serverSerial)
-
-	CreateSignedCert(root, CA, serverSerial, serverName, serverCACommonName)
-	CreateSignedCert(root, serverName, serverSerial, serverCertName, serverCertCommonName)
-
-	CreateSignedCert(root, CA, clientSerial, clientName, clientCACommonName)
-	CreateSignedCert(root, clientName, serverSerial, clientCertName, clientCertCommonName)
-
-	return clientServerKeyPairs{
-		serverCert: path.Join(root, fmt.Sprintf("%s-cert.pem", serverCertName)),
-		serverKey:  path.Join(root, fmt.Sprintf("%s-key.pem", serverCertName)),
-		serverCA:   path.Join(root, fmt.Sprintf("%s-cert.pem", serverName)),
-		clientCert: path.Join(root, fmt.Sprintf("%s-cert.pem", clientCertName)),
-		clientKey:  path.Join(root, fmt.Sprintf("%s-key.pem", clientCertName)),
-		clientCA:   path.Join(root, fmt.Sprintf("%s-cert.pem", clientName)),
-		serverName: serverCertCommonName,
-	}
-
-}
-
-func getServerConfig(keypairs clientServerKeyPairs) (*tls.Config, error) {
+func getServerConfig(keypairs ClientServerKeyPairs) (*tls.Config, error) {
 	return vttls.ServerConfig(
-		keypairs.clientCert,
-		keypairs.clientKey,
-		keypairs.serverCA)
+		keypairs.ClientCert,
+		keypairs.ClientKey,
+		keypairs.ServerCA)
 }
 
-func getClientConfig(keypairs clientServerKeyPairs) (*tls.Config, error) {
+func getClientConfig(keypairs ClientServerKeyPairs) (*tls.Config, error) {
 	return vttls.ClientConfig(
-		keypairs.clientCert,
-		keypairs.clientKey,
-		keypairs.serverCA,
-		keypairs.serverName)
+		keypairs.ClientCert,
+		keypairs.ClientKey,
+		keypairs.ServerCA,
+		keypairs.ServerName)
 }
 
 func TestServerTLSConfigCaching(t *testing.T) {
@@ -242,7 +191,7 @@ func TestClientTLSConfigCaching(t *testing.T) {
 	})
 }
 
-func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(clientServerKeyPairs) (*tls.Config, error), getCertPool func(tlsConfig *tls.Config) *x509.CertPool) {
+func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(ClientServerKeyPairs) (*tls.Config, error), getCertPool func(tlsConfig *tls.Config) *x509.CertPool) {
 	// Our test root.
 	root, err := ioutil.TempDir("", rootPrefix)
 	if err != nil {
@@ -252,8 +201,8 @@ func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(c
 
 	const configsToGenerate = 1
 
-	firstClientServerKeyPairs := createClientServerCertPairs(root)
-	secondClientServerKeyPairs := createClientServerCertPairs(root)
+	firstClientServerKeyPairs := CreateClientServerCertPairs(root)
+	secondClientServerKeyPairs := CreateClientServerCertPairs(root)
 
 	firstExpectedConfig, _ := generateConfig(firstClientServerKeyPairs)
 	secondExpectedConfig, _ := generateConfig(secondClientServerKeyPairs)