integration(ticdc): migration tidb-mysql-test to ticdc (#4694)

close #4693, ref #4719, ref #4720, ref #4721, ref #4722, ref #4723, ref #4724

.gitignore

@@ -20,7 +20,6 @@ _cgo_export.*
 _testmain.go
 
 *.exe
-*.test
 *.prof
 *.log
 

scripts/download-integration-test-binaries.sh

@@ -48,7 +48,7 @@ tiflash_download_url="${file_server_url}/download/builds/pingcap/tiflash/${branc
 minio_download_url="${file_server_url}/download/minio.tar.gz"
 go_ycsb_download_url="${file_server_url}/download/builds/pingcap/go-ycsb/test-br/go-ycsb"
 etcd_download_url="${file_server_url}/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz"
-sync_diff_inspector_url="${file_server_url}/download/builds/pingcap/cdc/new_sync_diff_inspector.tar.gz"
+sync_diff_inspector_url="${file_server_url}/download/builds/pingcap/cdc/sync_diff_inspector_hash-00998a9a_linux-amd64.tar.gz"
 jq_download_url="${file_server_url}/download/builds/pingcap/test/jq-1.6/jq-linux64"
 
 # Some temporary dir.

tests/integration_tests/tidb_mysql_test/README.md

@@ -0,0 +1,20 @@
+## Reference
+https://github.com/pingcap/tidb-test/tree/master/mysql_test
+https://github.com/pingcap/mysql-tester
+
+## Issue
+https://github.com/pingcap/tiflow/issues/4674
+
+## Description
+Migrate from TiDB mysql-test
+
+## Step
+1. Setup upstream and downstream cluster
+2. build mysql_test
+3. run all cases
+4. check data
+
+## Add New Test
+1. Add test and result file
+2. ./build.sh
+3. ./mysql_test or ./mysql_test xxx to check if test case works well locally (default host: 127.0.0.1:4000)

tests/integration_tests/tidb_mysql_test/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+# We use this repository temporaryly since mysql-tester will delete schema after every case
+# ref: https://github.com/pingcap/mysql-tester/blob/0ecb03c9cf5c0e19318bc5d0a38bcc59c8aab56d/src/main.go#L245
+# expected pr: https://github.com/pingcap/mysql-tester/pull/55
+GOBIN=$PWD go get github.com/maxshuang/mysql-tester/[email protected]
+mv src mysql_test
+echo -e "build mysql_test successfully"

tests/integration_tests/tidb_mysql_test/diff_config.toml

@@ -0,0 +1,29 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+    output-dir = "/tmp/tidb_cdc_test/tidb_mysql_test/sync_diff/output"
+
+    source-instances = ["tidb0"]
+
+    target-instance = "mysql1"
+
+    target-check-tables = ["alter_table.?*", "bigint.?*", "composite_index.?*", "concurrent_ddl.?*", "date_formats.?*", "drop.?*", "mysql_replace.?*"]
+
+[data-sources]
+[data-sources.tidb0]
+    host = "127.0.0.1"
+    port = 4000
+    user = "root"
+    password = ""
+
+[data-sources.mysql1]
+    host = "127.0.0.1"
+    port = 3306
+    user = "root"
+    password = ""

tests/integration_tests/tidb_mysql_test/r/alter_table.result

@@ -0,0 +1,149 @@
+drop table if exists warehouse;
+create table warehouse (
+id int PRIMARY KEY,
+w_idtiny tinyint not null,
+w_idmedium mediumint not null);
+ALTER TABLE warehouse ADD UNIQUE id_idx (w_idtiny);
+ALTER TABLE warehouse DROP INDEX id_idx;
+ALTER TABLE warehouse DROP PRIMARY KEY;
+ALTER TABLE warehouse DROP COLUMN w_idmedium;
+drop table if exists t;
+create table t (c1 int);
+alter table t add column c2 int;
+alter table t add column c3 int first;
+alter table t add column c4 int after c1;
+insert into t values (1, 2, 3, 4);
+select * from t;
+c3	c1	c4	c2
+1	2	3	4
+create table t1 (
+col1 int not null auto_increment primary key,
+col2 varchar(30) not null,
+col3 varchar (20) not null,
+col4 varchar(4) not null,
+col5 enum('PENDING', 'ACTIVE', 'DISABLED') not null,
+col6 int not null, to_be_deleted int);
+insert into t1 values (2,4,3,5,"PENDING",1,7);
+alter table t1 add column col4_5 varchar(20) not null after col4;
+alter table t1 add column col7 varchar(30) not null after col5;
+alter table t1 add column col8 datetime not null;
+alter table t1 drop column to_be_deleted;
+alter table t1 change column col2 fourth varchar(30) not null after col3;
+alter table t1 modify column col6 int not null first;
+select * from t1;
+col6	col1	col3	fourth	col4	col4_5	col5	col7	col8
+1	2	3	4	5		PENDING		0000-00-00 00:00:00
+CREATE TABLE t2 (
+GROUP_ID int(10) unsigned DEFAULT '0' NOT NULL,
+LANG_ID smallint(5) unsigned DEFAULT '0' NOT NULL,
+NAME varchar(80) DEFAULT '' NOT NULL,
+PRIMARY KEY (GROUP_ID,LANG_ID),
+KEY NAME (NAME));
+ALTER TABLE t2 CHANGE NAME NAME CHAR(80) not null;
+create table t3 (a int, b int, c int, d int, e int, f int, g int, h int,i int, primary key (a,b,c,d,e,f,g,i,h)) engine=MyISAM;
+alter table t3 modify a int;
+create table t4 (a int not null, b int not null, c int not null, d int not null, e int not null, f int not null, g int not null, h int not null,i int not null, primary key (a,b,c,d,e,f,g,i,h)) engine=MyISAM;
+CREATE TABLE t5 (s CHAR(8) BINARY);
+INSERT INTO t5 VALUES ('test');
+SELECT LENGTH(s) FROM t5;
+LENGTH(s)
+4
+ALTER TABLE t5 MODIFY s CHAR(10) BINARY;
+SELECT LENGTH(s) FROM t5;
+LENGTH(s)
+4
+CREATE TABLE t6 (s BINARY(8));
+INSERT INTO t6 VALUES ('test');
+SELECT LENGTH(s) FROM t6;
+LENGTH(s)
+8
+SELECT HEX(s) FROM t6;
+HEX(s)
+7465737400000000
+ALTER TABLE t6 MODIFY s BINARY(10);
+CREATE TABLE t7 (v VARCHAR(3), b INT);
+INSERT INTO t7 VALUES ('abc', 5);
+SELECT * FROM t7;
+v	b
+abc	5
+ALTER TABLE t7 MODIFY COLUMN v VARCHAR(4);
+SELECT * FROM t7;
+v	b
+abc	5
+CREATE TABLE t8 (f1 INT, f2 INT, f3 INT);
+INSERT INTO t8 VALUES (1, 2, NULL);
+SELECT * FROM t8;
+f1	f2	f3
+1	2	NULL
+ALTER TABLE t8 MODIFY COLUMN f3 INT AFTER f1;
+SELECT * FROM t8;
+f1	f3	f2
+1	NULL	2
+ALTER TABLE t8 MODIFY COLUMN f3 INT AFTER f2;
+SELECT * FROM t8;
+f1	f2	f3
+1	2	NULL
+#
+# Test for bug #12652385 - "61493: REORDERING COLUMNS TO POSITION
+#                           FIRST CAN CAUSE DATA TO BE CORRUPTED".
+#
+drop table if exists t9;
+# Use MyISAM engine as the fact that InnoDB doesn't support
+# in-place ALTER TABLE in cases when columns are being renamed
+# hides some bugs.
+create table t9 (i int, j int) engine=myisam;
+insert into t9 value (1, 2);
+# First, test for original problem described in the bug report.
+select * from t9;
+i	j
+1	2
+# Change of column order by the below ALTER TABLE statement should
+# affect both column names and column contents.
+alter table t9 modify column j int first;
+select * from t9;
+j	i
+2	1
+# Now test for similar problem with the same root.
+# The below ALTER TABLE should change not only the name but
+# also the value for the last column of the table.
+alter table t9 drop column i;
+alter table t9 add column k int default 0;
+select * from t9;
+j	k
+2	0
+# Clean-up.
+# Bug#11748057 (formerly known as 34972): ALTER TABLE statement doesn't
+#                                         identify correct column name.
+#
+CREATE TABLE t10 (c1 int unsigned , c2 char(100) not null default '');
+ALTER TABLE t10 ADD c3 char(16) NOT NULL DEFAULT '' AFTER c2;
+ALTER TABLE t10 MODIFY c2 char(100) NOT NULL DEFAULT '' AFTER c1;
+create table t11 (v varchar(32));
+insert into t11 values ('def'),('abc'),('hij'),('3r4f');
+select * from t11;
+v
+def
+abc
+hij
+3r4f
+alter table t11 change v v2 varchar(32);
+select * from t11;
+v2
+def
+abc
+hij
+3r4f
+alter table t11 change v2 v varchar(64);
+select * from t11;
+v
+def
+abc
+hij
+3r4f
+update t11 set v = 'lmn' where v = 'hij';
+select * from t11;
+v
+def
+abc
+lmn
+3r4f