utils(dm): fix get tables without using quote schema name (#5896)

close #5895

dm/dm/master/openapi_view.go

@@ -22,10 +22,10 @@ import (
 	"net/http"
 	"net/http/httputil"
 
-	"github.com/pingcap/failpoint"
-
 	ginmiddleware "github.com/deepmap/oapi-codegen/pkg/gin-middleware"
 	"github.com/gin-gonic/gin"
+	"github.com/pingcap/failpoint"
+	"github.com/pingcap/tidb/util/dbutil"
 	"go.uber.org/zap"
 
 	"github.com/pingcap/tiflow/dm/dm/config"
@@ -36,7 +36,6 @@ import (
 	"github.com/pingcap/tiflow/dm/pkg/ha"
 	"github.com/pingcap/tiflow/dm/pkg/log"
 	"github.com/pingcap/tiflow/dm/pkg/terror"
-	"github.com/pingcap/tiflow/dm/pkg/utils"
 )
 
 const (
@@ -374,7 +373,7 @@ func (s *Server) DMAPIGetSourceSchemaList(c *gin.Context, sourceName string) {
 		return
 	}
 	defer baseDB.Close()
-	schemaList, err := utils.GetSchemaList(c.Request.Context(), baseDB.DB)
+	schemaList, err := dbutil.GetSchemas(c.Request.Context(), baseDB.DB)
 	if err != nil {
 		_ = c.Error(err)
 		return
@@ -390,7 +389,7 @@ func (s *Server) DMAPIGetSourceTableList(c *gin.Context, sourceName string, sche
 		return
 	}
 	defer baseDB.Close()
-	tableList, err := utils.GetTableList(c.Request.Context(), baseDB.DB, schemaName)
+	tableList, err := dbutil.GetTables(c.Request.Context(), baseDB.DB, schemaName)
 	if err != nil {
 		_ = c.Error(err)
 		return

dm/dm/master/openapi_view_test.go

@@ -710,7 +710,8 @@ func (s *OpenAPIViewSuite) TestSourceAPI() {
 	_, mockDB, err = conn.InitMockDBFull()
 	s.NoError(err)
 	tableName := "CHARACTER_SETS"
-	mockDB.ExpectQuery("SHOW TABLES FROM " + schemaName).WillReturnRows(sqlmock.NewRows([]string{"Tables_in_information_schema"}).AddRow(tableName))
+	mockDB.ExpectQuery("SHOW FULL TABLES IN `information_schema` WHERE Table_Type != 'VIEW';").WillReturnRows(
+		sqlmock.NewRows([]string{"Tables_in_information_schema", "Table_type"}).AddRow(tableName, "BASE TABLE"))
 	tableURL := fmt.Sprintf("%s/%s/schemas/%s", baseURL, source1.SourceName, schemaName)
 	result = testutil.NewRequest().Get(tableURL).GoWithHTTPHandler(s.T(), s1.openapiHandles)
 	s.Equal(http.StatusOK, result.Code())

dm/pkg/checker/onlineddl.go

@@ -17,9 +17,9 @@ import (
 	"context"
 	"database/sql"
 
+	"github.com/pingcap/tidb/util/dbutil"
 	"github.com/pingcap/tidb/util/filter"
 
-	"github.com/pingcap/tiflow/dm/pkg/utils"
 	onlineddl "github.com/pingcap/tiflow/dm/syncer/online-ddl-tools"
 )
 
@@ -48,7 +48,7 @@ func (c *OnlineDDLChecker) Check(ctx context.Context) *Result {
 	}
 
 	for schema := range c.checkSchemas {
-		tableList, err := utils.GetTableList(ctx, c.db, schema)
+		tableList, err := dbutil.GetTables(ctx, c.db, schema)
 		if err != nil {
 			markCheckError(r, err)
 			return r

dm/pkg/utils/db.go

@@ -405,49 +405,6 @@ func GetServerUnixTS(ctx context.Context, db *sql.DB) (int64, error) {
 	return ts, err
 }
 
-// GetSchemaList gets db schema list with `SHOW DATABASES`.
-func GetSchemaList(ctx context.Context, db *sql.DB) ([]string, error) {
-	schemaList := []string{}
-	rows, err := db.QueryContext(ctx, "SHOW DATABASES")
-	if err != nil {
-		return schemaList, terror.DBErrorAdapt(err, terror.ErrDBDriverError)
-	}
-	if rows.Err() != nil {
-		return nil, terror.DBErrorAdapt(rows.Err(), terror.ErrDBDriverError)
-	}
-	defer rows.Close()
-	for rows.Next() {
-		var schema string
-		if scanErr := rows.Scan(&schema); scanErr != nil {
-			return nil, terror.DBErrorAdapt(scanErr, terror.ErrDBDriverError)
-		}
-		schemaList = append(schemaList, schema)
-	}
-	return schemaList, err
-}
-
-// GetTableList gets db schema list with `SHOW TABLES FROM`.
-func GetTableList(ctx context.Context, db *sql.DB, schemaName string) ([]string, error) {
-	tableList := []string{}
-	sql := "SHOW TABLES FROM " + schemaName
-	rows, err := db.QueryContext(ctx, sql)
-	if err != nil {
-		return tableList, terror.DBErrorAdapt(err, terror.ErrDBDriverError)
-	}
-	if rows.Err() != nil {
-		return nil, terror.DBErrorAdapt(rows.Err(), terror.ErrDBDriverError)
-	}
-	defer rows.Close()
-	for rows.Next() {
-		var table string
-		if scanErr := rows.Scan(&table); scanErr != nil {
-			return nil, terror.DBErrorAdapt(scanErr, terror.ErrDBDriverError)
-		}
-		tableList = append(tableList, table)
-	}
-	return tableList, err
-}
-
 // GetMariaDBUUID gets equivalent `server_uuid` for MariaDB
 // `gtid_domain_id` joined `server_id` with domainServerIDSeparator.
 func GetMariaDBUUID(ctx context.Context, db *sql.DB) (string, error) {

dm/pkg/utils/db_test.go

@@ -215,40 +215,6 @@ func (t *testDBSuite) TestGetServerUnixTS(c *C) {
 	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
 
-func (t *testDBSuite) TestGetSchemaList(c *C) {
-	ctx := context.Background()
-
-	db, mock, err := sqlmock.New()
-	c.Assert(err, IsNil)
-
-	schemaName := "information_schema"
-	rows := sqlmock.NewRows([]string{"Database"}).AddRow(schemaName)
-	mock.ExpectQuery("SHOW DATABASES").WillReturnRows(rows)
-
-	schemaList, err := GetSchemaList(ctx, db)
-	c.Assert(err, IsNil)
-	c.Assert(schemaName, Equals, schemaList[0])
-	c.Assert(mock.ExpectationsWereMet(), IsNil)
-}
-
-func (t *testDBSuite) TestGetTableList(c *C) {
-	ctx := context.Background()
-
-	db, mock, err := sqlmock.New()
-	c.Assert(err, IsNil)
-
-	schemaName := "information_schema"
-	tableName := "CHARACTER_SETS"
-	sql := "SHOW TABLES FROM " + schemaName
-	rows := sqlmock.NewRows([]string{"Tables_in_information_schema"}).AddRow(tableName)
-	mock.ExpectQuery(sql).WillReturnRows(rows)
-
-	tableList, err := GetTableList(ctx, db, schemaName)
-	c.Assert(err, IsNil)
-	c.Assert(tableName, Equals, tableList[0])
-	c.Assert(mock.ExpectationsWereMet(), IsNil)
-}
-
 func (t *testDBSuite) TestGetParser(c *C) {
 	ctx, cancel := context.WithTimeout(context.Background(), DefaultDBTimeout)
 	defer cancel()

dm/tests/check_task/conf/task-sharding.yaml

@@ -11,7 +11,7 @@ target-database:       # 下游数据库实例配置
 ## ******** 功能配置集 **********
 block-allow-list:        # 上游数据库实例匹配的表的 block-allow-list 过滤规则集，如果 DM 版本 <= v2.0.0-beta.2 则使用 black-white-list
   bw-rule-1:             # 黑白名单配置的名称
-    do-dbs: ["checktask"] # 迁移哪些库
+    do-dbs: ["check-task"] # 迁移哪些库
 
 # ----------- 实例配置 -----------
 mysql-instances:
@@ -22,9 +22,9 @@ mysql-instances:
 
 routes:
   rule1:
-    schema-pattern: "checktask"
+    schema-pattern: "check-task"
     table-pattern: "t*"
-    target-schema: "checktask"
+    target-schema: "check-task"
     target-table: "t"
 
 mydumpers:

dm/tests/check_task/data/db1.prepare.sql

@@ -1,4 +1,6 @@
-use checktask;
+-- test the db name that must be quoted are working properly;
+use `check-task`;
+
 create table t1(c int primary key);
 
 create table t2(c int primary key, c2 int);
@@ -9,4 +11,4 @@ create table t4(c int primary key, c4 int);
 
 create table t5(c int primary key, c5 int);
 
-create table t6(c int primary key, c6 int);
+create table t6(c int primary key, c6 int);

dm/tests/check_task/run.sh

@@ -14,12 +14,6 @@ function prepare_incompatible_tables() {
 	done
 }
 
-function prepare_many_tables() {
-	run_sql_both_source "drop database if exists checktask"
-	run_sql_both_source "create database if not exists checktask"
-	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
-}
-
 function prepare() {
 	run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
 	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
@@ -36,7 +30,10 @@ function test_check_task_fail_no_block() {
 }
 
 function test_check_task_fail_no_block_forsharding() {
-	prepare_many_tables
+	run_sql_both_source "drop database if exists \`check-task\`"
+	run_sql_both_source "create database if not exists \`check-task\`"
+	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
+
 	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
 		"check-task $cur/conf/task-sharding.yaml" \
 		"\"state\": \"fail\"" 1
@@ -48,8 +45,10 @@ function run() {
 	test_check_task_fail_no_block_forsharding
 }
 
+cleanup_data check-task
 cleanup_data checktask
 cleanup_process $*
 run $*
 cleanup_process $*
 cleanup_data checktask
+cleanup_data check-task

dm/tests/openapi/run.sh

@@ -81,9 +81,17 @@ function test_source() {
 		"\"worker\": \"worker2\"" 1
 
 	init_noshard_data # init table in database openapi for test get schema and table
+
 	# test get source schemas and tables
 	openapi_source_check "get_source_schemas_and_tables_success" "mysql-01" "openapi" "t1"
 
+	# test the db name that must be quoted are working properly
+	must_quote_db_name="\`db-name\`"
+	run_sql_source1 "create database if not exists $must_quote_db_name"
+	run_sql_source1 "create table $must_quote_db_name.t1 (i TINYINT, j INT UNIQUE KEY)"
+	openapi_source_check "get_source_schemas_and_tables_success" "mysql-01" "db-name" "t1"
+	run_sql_source1 "drop database $must_quote_db_name"
+
 	# delete source success
 	openapi_source_check "delete_source_success" "mysql-01"