Merge pull request #4803 from planetscale/ds-xb-mysql80

test xtrabackup with mysql 8.0 and percona 8.0
vitessio · Apr 16, 2019 · d2d8820 · d2d8820
2 parents b383d00 + 569e90e
commit d2d8820
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 83 deletions.
diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80
@@ -1,6 +1,6 @@
 FROM vitess/bootstrap:common
 
-# Install MySQL 5.7
+# Install MySQL 8.0
 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 8C718D3B5072E1F5 && break; done && \
     add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-8.0' && \
     apt-get update -y && \

diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80
@@ -15,11 +15,17 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
 	percona-server-tokudb \
 	percona-server-rocksdb \
 	bzip2 \
-    && rm -rf /var/lib/apt/lists/*	
+	libdbd-mysql-perl \
+	rsync \
+	libev4 \
+    && rm -rf /var/lib/apt/lists/* \
+    && wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-8.0.4/binary/debian/stretch/x86_64/percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb \
+    && dpkg -i percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb \
+    && rm -f percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
-ENV MYSQL_FLAVOR MySQL56
+ENV MYSQL_FLAVOR MySQL80
 USER vitess
 RUN ./bootstrap.sh
diff --git a/go/vt/mysqlctl/xtrabackupengine.go b/go/vt/mysqlctl/xtrabackupengine.go
@@ -189,12 +189,12 @@ func (be *XtrabackupEngine) ExecuteBackup(ctx context.Context, cnf *Mycnf, mysql
 	if err != nil {
 		return false, vterrors.Wrap(err, "backup failed while reading command output")
 	}
-	execErr := backupCmd.Wait()
-	if execErr != nil {
-		return false, vterrors.Wrap(err, "xtrabackup failed with error")
-	}
+	err = backupCmd.Wait()
 	output := string(stderrOutput)
 	logger.Infof("Xtrabackup backup command output: %v", output)
+	if err != nil {
+		return false, vterrors.Wrap(err, "xtrabackup failed with error")
+	}
 
 	replicationPosition, rerr := findReplicationPosition(output, flavor, logger)
 	if rerr != nil {
@@ -470,7 +470,8 @@ func findReplicationPosition(input, flavor string, logger logutil.Logger) (mysql
 		}
 	}
 	position := ""
-	if index != -1 {
+	// asserts that xtrabackup output comes with GTIDs in the format we expect
+	if index != -1 && index < len(substrs) {
 		// since we are extracting this from the log, it contains newlines
 		// replace them with a single space to match the SET GLOBAL gtid_purged command in xtrabackup_slave_info
 		position = strings.Replace(substrs[index], "\n", " ", -1)

diff --git a/test/backup.py b/test/backup.py
@@ -29,23 +29,23 @@
 use_mysqlctld = False
 use_xtrabackup = False
 stream_mode = 'tar'
-xtrabackup_args = ['-backup_engine_implementation',
-                   'xtrabackup',
-                   '-xtrabackup_stream_mode',
-                   stream_mode,
-                   '-xtrabackup_user=vt_dba',
-                   '-xtrabackup_backup_flags',
-                   '--password=VtDbaPass']
-
 tablet_master = None
 tablet_replica1 = None
 tablet_replica2 = None
-
+xtrabackup_args = []
 new_init_db = ''
 db_credentials_file = ''
 
 
 def setUpModule():
+  xtrabackup_args = ['-backup_engine_implementation',
+                   'xtrabackup',
+                   '-xtrabackup_stream_mode',
+                   stream_mode,
+                   '-xtrabackup_user=vt_dba',
+                   '-xtrabackup_backup_flags',
+                   '--password=VtDbaPass']
+
   global new_init_db, db_credentials_file
   global tablet_master, tablet_replica1, tablet_replica2
 
@@ -59,21 +59,6 @@ def setUpModule():
   try:
     environment.topo_server().setup()
 
-    # Determine which column is used for user passwords in this MySQL version.
-    proc = tablet_master.init_mysql()
-    if use_mysqlctld:
-      tablet_master.wait_for_mysqlctl_socket()
-    else:
-      utils.wait_procs([proc])
-    try:
-      tablet_master.mquery('mysql', 'select password from mysql.user limit 0',
-                           user='root')
-      password_col = 'password'
-    except MySQLdb.DatabaseError:
-      password_col = 'authentication_string'
-    utils.wait_procs([tablet_master.teardown_mysql()])
-    tablet_master.remove_tree(ignore_options=True)
-
     # Create a new init_db.sql file that sets up passwords for all users.
     # Then we use a db-credentials-file with the passwords.
     new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
@@ -83,20 +68,14 @@ def setUpModule():
       fd.write(init_db)
       fd.write('''
 # Set real passwords for all users.
-UPDATE mysql.user SET %s = PASSWORD('RootPass')
-  WHERE User = 'root' AND Host = 'localhost';
-UPDATE mysql.user SET %s = PASSWORD('VtDbaPass')
-  WHERE User = 'vt_dba' AND Host = 'localhost';
-UPDATE mysql.user SET %s = PASSWORD('VtAppPass')
-  WHERE User = 'vt_app' AND Host = 'localhost';
-UPDATE mysql.user SET %s = PASSWORD('VtAllprivsPass')
-  WHERE User = 'vt_allprivs' AND Host = 'localhost';
-UPDATE mysql.user SET %s = PASSWORD('VtReplPass')
-  WHERE User = 'vt_repl' AND Host = '%%';
-UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass')
-  WHERE User = 'vt_filtered' AND Host = 'localhost';
+ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass';
+ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass';
+ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass';
+ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllPrivsPass';
+ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass';
+ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass';
 FLUSH PRIVILEGES;
-''' % tuple([password_col] * 6))
+''')
     credentials = {
         'vt_dba': ['VtDbaPass'],
         'vt_app': ['VtAppPass'],

diff --git a/test/xtrabackup_xtra.py b/test/xtrabackup_xtra.py
@@ -28,15 +28,8 @@
 
 use_mysqlctld = False
 use_xtrabackup = True
-stream_mode = 'tar'
-xtrabackup_args = ['-backup_engine_implementation',
-                   'xtrabackup',
-                   '-xtrabackup_stream_mode',
-                   stream_mode,
-                   '-xtrabackup_user=vt_dba',
-                   '-xtrabackup_backup_flags',
-                   '--password=VtDbaPass']
-
+stream_mode = 'xbstream'
+xtrabackup_args = []
 tablet_master = None
 tablet_replica1 = None
 tablet_replica2 = None
@@ -46,6 +39,13 @@
 
 
 def setUpModule():
+  xtrabackup_args = ['-backup_engine_implementation',
+                   'xtrabackup',
+                   '-xtrabackup_stream_mode',
+                   stream_mode,
+                   '-xtrabackup_user=vt_dba',
+                   '-xtrabackup_backup_flags',
+                   '--password=VtDbaPass']
   global new_init_db, db_credentials_file
   global tablet_master, tablet_replica1, tablet_replica2
 
@@ -59,21 +59,6 @@ def setUpModule():
   try:
     environment.topo_server().setup()
 
-    # Determine which column is used for user passwords in this MySQL version.
-    proc = tablet_master.init_mysql()
-    if use_mysqlctld:
-      tablet_master.wait_for_mysqlctl_socket()
-    else:
-      utils.wait_procs([proc])
-    try:
-      tablet_master.mquery('mysql', 'select password from mysql.user limit 0',
-                           user='root')
-      password_col = 'password'
-    except MySQLdb.DatabaseError:
-      password_col = 'authentication_string'
-    utils.wait_procs([tablet_master.teardown_mysql()])
-    tablet_master.remove_tree(ignore_options=True)
-
     # Create a new init_db.sql file that sets up passwords for all users.
     # Then we use a db-credentials-file with the passwords.
     new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
@@ -82,21 +67,15 @@ def setUpModule():
     with open(new_init_db, 'w') as fd:
       fd.write(init_db)
       fd.write('''
-# Set real passwords for all users except vt_backup
-UPDATE mysql.user SET %s = PASSWORD('RootPass')
-  WHERE User = 'root' AND Host = 'localhost';
-UPDATE mysql.user SET %s = PASSWORD('VtDbaPass')
-  WHERE User = 'vt_dba' AND Host = 'localhost';
-UPDATE mysql.user SET %s = PASSWORD('VtAppPass')
-  WHERE User = 'vt_app' AND Host = 'localhost';
-UPDATE mysql.user SET %s = PASSWORD('VtAllprivsPass')
-  WHERE User = 'vt_allprivs' AND Host = 'localhost';
-UPDATE mysql.user SET %s = PASSWORD('VtReplPass')
-  WHERE User = 'vt_repl' AND Host = '%%';
-UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass')
-  WHERE User = 'vt_filtered' AND Host = 'localhost';
+# Set real passwords for all users.
+ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass';
+ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass';
+ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass';
+ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllPrivsPass';
+ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass';
+ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass';
 FLUSH PRIVILEGES;
-''' % tuple([password_col] * 6))
+''')
     credentials = {
         'vt_dba': ['VtDbaPass'],
         'vt_app': ['VtAppPass'],