From cdbab20e809c2d30b1c9be80e343eb8882456cd2 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 20 Dec 2021 19:49:46 +0800 Subject: [PATCH 1/3] cherry pick #30439 to release-5.3 Signed-off-by: ti-srebot --- br/pkg/lightning/common/storage_unix.go | 7 +- br/pkg/lightning/common/storage_windows.go | 5 + br/pkg/lightning/restore/check_info.go | 36 +- br/pkg/lightning/restore/check_info_test.go | 455 ++++++++++++++++++++ br/pkg/lightning/restore/check_template.go | 21 +- br/pkg/lightning/restore/restore.go | 3 +- 6 files changed, 495 insertions(+), 32 deletions(-) create mode 100644 br/pkg/lightning/restore/check_info_test.go diff --git a/br/pkg/lightning/common/storage_unix.go b/br/pkg/lightning/common/storage_unix.go index ba22e92354ceb..7e602cbe58eec 100644 --- a/br/pkg/lightning/common/storage_unix.go +++ b/br/pkg/lightning/common/storage_unix.go @@ -23,13 +23,18 @@ import ( "syscall" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "golang.org/x/sys/unix" ) // GetStorageSize gets storage's capacity and available size func GetStorageSize(dir string) (size StorageSize, err error) { - var stat unix.Statfs_t + failpoint.Inject("GetStorageSize", func(val failpoint.Value) { + injectedSize := val.(int) + failpoint.Return(StorageSize{Capacity: uint64(injectedSize), Available: uint64(injectedSize)}, nil) + }) + var stat unix.Statfs_t err = unix.Statfs(dir, &stat) if err != nil { return size, errors.Annotatef(err, "cannot get disk capacity at %s", dir) diff --git a/br/pkg/lightning/common/storage_windows.go b/br/pkg/lightning/common/storage_windows.go index 21a2398ad66c3..a95e8f8eeebfc 100644 --- a/br/pkg/lightning/common/storage_windows.go +++ b/br/pkg/lightning/common/storage_windows.go @@ -23,6 +23,7 @@ import ( "unsafe" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" ) var ( @@ -32,6 +33,10 @@ var ( // GetStorageSize gets storage's capacity and available size func GetStorageSize(dir string) (size StorageSize, err error) { + failpoint.Inject("GetStorageSize", func(val failpoint.Value) { + injectedSize := val.(int) + failpoint.Return(StorageSize{Capacity: uint64(injectedSize), Available: uint64(injectedSize)}, nil) + }) r, _, e := getDiskFreeSpaceExW.Call( uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(dir))), uintptr(unsafe.Pointer(&size.Available)), diff --git a/br/pkg/lightning/restore/check_info.go b/br/pkg/lightning/restore/check_info.go index 1b86ee482f362..d597b6e2646fb 100644 --- a/br/pkg/lightning/restore/check_info.go +++ b/br/pkg/lightning/restore/check_info.go @@ -454,33 +454,31 @@ func (rc *Controller) localResource(sourceSize int64) error { if err != nil { return errors.Trace(err) } - localAvailable := storageSize.Available + localAvailable := int64(storageSize.Available) var message string var passed bool switch { - case localAvailable > uint64(sourceSize): + case localAvailable > sourceSize: message = fmt.Sprintf("local disk resources are rich, estimate sorted data size %s, local available is %s", units.BytesSize(float64(sourceSize)), units.BytesSize(float64(localAvailable))) passed = true + case int64(rc.cfg.TikvImporter.DiskQuota) > localAvailable: + message = fmt.Sprintf("local disk space may not enough to finish import, estimate sorted data size is %s,"+ + " but local available is %s, please set `tikv-importer.disk-quota` to a smaller value than %s"+ + " or change `mydumper.sorted-kv-dir` to another disk with enough space to finish imports", + units.BytesSize(float64(sourceSize)), + units.BytesSize(float64(localAvailable)), units.BytesSize(float64(localAvailable))) + passed = false + log.L().Error(message) default: - if int64(rc.cfg.TikvImporter.DiskQuota) > int64(localAvailable) { - message = fmt.Sprintf("local disk space may not enough to finish import"+ - "estimate sorted data size is %s, but local available is %s,"+ - "you need a smaller number for tikv-importer.disk-quota (%s) to finish imports", - units.BytesSize(float64(sourceSize)), - units.BytesSize(float64(localAvailable)), units.BytesSize(float64(rc.cfg.TikvImporter.DiskQuota))) - passed = false - log.L().Error(message) - } else { - message = fmt.Sprintf("local disk space may not enough to finish import, "+ - "estimate sorted data size is %s, but local available is %s,"+ - "we will use disk-quota (size: %s) to finish imports, which may slow down import", - units.BytesSize(float64(sourceSize)), - units.BytesSize(float64(localAvailable)), units.BytesSize(float64(rc.cfg.TikvImporter.DiskQuota))) - passed = true - log.L().Warn(message) - } + message = fmt.Sprintf("local disk space may not enough to finish import, "+ + "estimate sorted data size is %s, but local available is %s,"+ + "we will use disk-quota (size: %s) to finish imports, which may slow down import", + units.BytesSize(float64(sourceSize)), + units.BytesSize(float64(localAvailable)), units.BytesSize(float64(rc.cfg.TikvImporter.DiskQuota))) + passed = true + log.L().Warn(message) } rc.checkTemplate.Collect(Critical, passed, message) return nil diff --git a/br/pkg/lightning/restore/check_info_test.go b/br/pkg/lightning/restore/check_info_test.go new file mode 100644 index 0000000000000..ccc4aa74c0c28 --- /dev/null +++ b/br/pkg/lightning/restore/check_info_test.go @@ -0,0 +1,455 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package restore + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/pingcap/check" + "github.com/pingcap/failpoint" + + "github.com/pingcap/tidb/br/pkg/lightning/checkpoints" + "github.com/pingcap/tidb/br/pkg/lightning/config" + "github.com/pingcap/tidb/br/pkg/lightning/mydump" + "github.com/pingcap/tidb/br/pkg/lightning/worker" + "github.com/pingcap/tidb/br/pkg/storage" + "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/parser/mysql" + tmock "github.com/pingcap/tidb/util/mock" +) + +var _ = Suite(&checkInfoSuite{}) + +type checkInfoSuite struct{} + +const passed CheckType = "pass" + +func (s *checkInfoSuite) TestCheckCSVHeader(c *C) { + dir := c.MkDir() + ctx := context.Background() + mockStore, err := storage.NewLocalStorage(dir) + c.Assert(err, IsNil) + + type tableSource struct { + Name string + SQL string + Sources []string + } + + cases := []struct { + ignoreColumns []*config.IgnoreColumns + // empty msg means check pass + level CheckType + Sources map[string][]*tableSource + }{ + + { + nil, + + passed, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8))", + []string{ + "aa,b\r\n", + }, + }, + }, + }, + }, + { + nil, + + passed, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8))", + []string{ + "a,b\r\ntest1,test2\r\n", + "aa,b\r\n", + }, + }, + }, + }, + }, + { + nil, + + Warn, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8))", + []string{ + "a,b\r\n", + }, + }, + }, + }, + }, + { + nil, + + Warn, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8))", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\n", + }, + }, + }, + }, + }, + { + nil, + + Warn, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`))", + []string{ + "a,b\r\ntest1,test2\r\n", + }, + }, + }, + }, + }, + { + nil, + + Critical, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`))", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\r\n", + }, + }, + }, + }, + }, + // ignore primary key, should still be warn + { + []*config.IgnoreColumns{ + { + DB: "db", + Table: "tbl1", + Columns: []string{"a"}, + }, + }, + Warn, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`))", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\r\n", + }, + }, + }, + }, + }, + // ignore primary key, but has other unique key + { + []*config.IgnoreColumns{ + { + DB: "db", + Table: "tbl1", + Columns: []string{"a"}, + }, + }, + Critical, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`), unique key uk (`b`))", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\r\n", + }, + }, + }, + }, + }, + // ignore primary key, non other unique key + { + []*config.IgnoreColumns{ + { + DB: "db", + Table: "tbl1", + Columns: []string{"a"}, + }, + }, + Warn, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`), KEY idx_b (`b`))", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\r\n", + }, + }, + }, + }, + }, + // non unique key, but data type inconsistent + { + nil, + Critical, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a bigint, b varchar(8));", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\r\n", + }, + }, + }, + }, + }, + // non unique key, but ignore inconsistent field + { + []*config.IgnoreColumns{ + { + DB: "db", + Table: "tbl1", + Columns: []string{"a"}, + }, + }, + Warn, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a bigint, b varchar(8));", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\r\n", + }, + }, + }, + }, + }, + // multiple tables, test the choose priority + { + nil, + Critical, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(8), b varchar(8));", + []string{ + "a,b\r\ntest1,test2\r\n", + }, + }, + { + "tbl2", + "create table tbl1 (a varchar(8) primary key, b varchar(8));", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\r\n", + }, + }, + }, + }, + }, + { + nil, + Critical, + map[string][]*tableSource{ + "db": { + { + "tbl1", + "create table tbl1 (a varchar(8), b varchar(8));", + []string{ + "a,b\r\ntest1,test2\r\n", + }, + }, + }, + "db2": { + { + "tbl2", + "create table tbl1 (a bigint, b varchar(8));", + []string{ + "a,b\r\ntest1,test2\r\n", + "a,b\r\ntest3,test4\r\n", + }, + }, + }, + }, + }, + } + + cfg := &config.Config{ + Mydumper: config.MydumperRuntime{ + ReadBlockSize: config.ReadBlockSize, + CSV: config.CSVConfig{ + Separator: ",", + Delimiter: `"`, + Header: false, + NotNull: false, + Null: `\N`, + BackslashEscape: true, + TrimLastSep: false, + }, + }, + } + rc := &Controller{ + cfg: cfg, + store: mockStore, + ioWorkers: worker.NewPool(context.Background(), 1, "io"), + } + + p := parser.New() + p.SetSQLMode(mysql.ModeANSIQuotes) + se := tmock.NewContext() + + for _, ca := range cases { + rc.checkTemplate = NewSimpleTemplate() + cfg.Mydumper.IgnoreColumns = ca.ignoreColumns + rc.dbInfos = make(map[string]*checkpoints.TidbDBInfo) + + dbMetas := make([]*mydump.MDDatabaseMeta, 0) + for db, tbls := range ca.Sources { + tblMetas := make([]*mydump.MDTableMeta, 0, len(tbls)) + dbInfo := &checkpoints.TidbDBInfo{ + Name: db, + Tables: make(map[string]*checkpoints.TidbTableInfo), + } + rc.dbInfos[db] = dbInfo + + for _, tbl := range tbls { + node, err := p.ParseOneStmt(tbl.SQL, "", "") + c.Assert(err, IsNil) + core, err := ddl.MockTableInfo(se, node.(*ast.CreateTableStmt), 0xabcdef) + c.Assert(err, IsNil) + core.State = model.StatePublic + dbInfo.Tables[tbl.Name] = &checkpoints.TidbTableInfo{ + ID: core.ID, + DB: db, + Name: tbl.Name, + Core: core, + } + + fileInfos := make([]mydump.FileInfo, 0, len(tbl.Sources)) + for i, s := range tbl.Sources { + fileName := fmt.Sprintf("%s.%s.%d.csv", db, tbl.Name, i) + err = os.WriteFile(filepath.Join(dir, fileName), []byte(s), 0o644) + c.Assert(err, IsNil) + fileInfos = append(fileInfos, mydump.FileInfo{ + FileMeta: mydump.SourceFileMeta{ + Path: fileName, + Type: mydump.SourceTypeCSV, + FileSize: int64(len(s)), + }, + }) + } + tblMetas = append(tblMetas, &mydump.MDTableMeta{ + DB: db, + Name: tbl.Name, + DataFiles: fileInfos, + }) + } + dbMetas = append(dbMetas, &mydump.MDDatabaseMeta{ + Name: db, + Tables: tblMetas, + }) + } + + err := rc.checkCSVHeader(ctx, dbMetas) + c.Assert(err, IsNil) + if ca.level != passed { + c.Assert(rc.checkTemplate.FailedCount(ca.level), Equals, 1) + } + } +} + +func (s *checkInfoSuite) TestLocalResource(c *C) { + dir := c.MkDir() + mockStore, err := storage.NewLocalStorage(dir) + c.Assert(err, IsNil) + + err = failpoint.Enable("github.com/pingcap/tidb/br/pkg/lightning/common/GetStorageSize", "return(2048)") + c.Assert(err, IsNil) + defer func() { + _ = failpoint.Disable("github.com/pingcap/tidb/br/pkg/lightning/common/GetStorageSize") + }() + + cfg := config.NewConfig() + cfg.Mydumper.SourceDir = dir + cfg.TikvImporter.SortedKVDir = dir + cfg.TikvImporter.Backend = "local" + rc := &Controller{ + cfg: cfg, + store: mockStore, + ioWorkers: worker.NewPool(context.Background(), 1, "io"), + } + + // 1. source-size is smaller than disk-size, won't trigger error information + rc.checkTemplate = NewSimpleTemplate() + err = rc.localResource(1000) + c.Assert(err, IsNil) + tmpl := rc.checkTemplate.(*SimpleTemplate) + c.Assert(tmpl.warnFailedCount, Equals, 1) + c.Assert(tmpl.criticalFailedCount, Equals, 0) + c.Assert(tmpl.normalMsgs[1], Matches, "local disk resources are rich, estimate sorted data size 1000B, local available is 2KiB") + + // 2. source-size is bigger than disk-size, with default disk-quota will trigger a critical error + rc.checkTemplate = NewSimpleTemplate() + err = rc.localResource(4096) + c.Assert(err, IsNil) + tmpl = rc.checkTemplate.(*SimpleTemplate) + c.Assert(tmpl.warnFailedCount, Equals, 1) + c.Assert(tmpl.criticalFailedCount, Equals, 1) + c.Assert(tmpl.criticalMsgs[0], Matches, "local disk space may not enough to finish import, estimate sorted data size is 4KiB, but local available is 2KiB, please set `tikv-importer.disk-quota` to a smaller value than 2KiB or change `mydumper.sorted-kv-dir` to another disk with enough space to finish imports") + + // 3. source-size is bigger than disk-size, with a vaild disk-quota will trigger a warning + rc.checkTemplate = NewSimpleTemplate() + rc.cfg.TikvImporter.DiskQuota = config.ByteSize(1024) + err = rc.localResource(4096) + c.Assert(err, IsNil) + tmpl = rc.checkTemplate.(*SimpleTemplate) + c.Assert(tmpl.warnFailedCount, Equals, 1) + c.Assert(tmpl.criticalFailedCount, Equals, 0) + c.Assert(tmpl.normalMsgs[1], Matches, "local disk space may not enough to finish import, estimate sorted data size is 4KiB, but local available is 2KiB,we will use disk-quota \\(size: 1KiB\\) to finish imports, which may slow down import") +} diff --git a/br/pkg/lightning/restore/check_template.go b/br/pkg/lightning/restore/check_template.go index 3fb8c22904caa..f38e23aa00f8e 100644 --- a/br/pkg/lightning/restore/check_template.go +++ b/br/pkg/lightning/restore/check_template.go @@ -51,7 +51,8 @@ type SimpleTemplate struct { count int warnFailedCount int criticalFailedCount int - failedMsg []string + normalMsgs []string // only used in unit test now + criticalMsgs []string t table.Writer } @@ -65,16 +66,12 @@ func NewSimpleTemplate() Template { {Name: "Passed", WidthMax: 6}, }) return &SimpleTemplate{ - 0, - 0, - 0, - make([]string, 0), - t, + t: t, } } func (c *SimpleTemplate) FailedMsg() string { - return strings.Join(c.failedMsg, ";\n") + return strings.Join(c.criticalMsgs, ";\n") } func (c *SimpleTemplate) Collect(t CheckType, passed bool, msg string) { @@ -87,7 +84,11 @@ func (c *SimpleTemplate) Collect(t CheckType, passed bool, msg string) { c.warnFailedCount++ } } - c.failedMsg = append(c.failedMsg, msg) + if !passed && t == Critical { + c.criticalMsgs = append(c.criticalMsgs, msg) + } else { + c.normalMsgs = append(c.normalMsgs, msg) + } c.t.AppendRow(table.Row{c.count, msg, t, passed}) c.t.AppendSeparator() } @@ -108,7 +109,7 @@ func (c *SimpleTemplate) FailedCount(t CheckType) int { func (c *SimpleTemplate) Output() string { c.t.SetAllowedRowLength(170) - c.t.SetRowPainter(table.RowPainter(func(row table.Row) text.Colors { + c.t.SetRowPainter(func(row table.Row) text.Colors { if passed, ok := row[3].(bool); ok { if !passed { if typ, ok := row[2].(CheckType); ok { @@ -122,7 +123,7 @@ func (c *SimpleTemplate) Output() string { } } return nil - })) + }) res := c.t.Render() summary := "\n" if c.criticalFailedCount > 0 { diff --git a/br/pkg/lightning/restore/restore.go b/br/pkg/lightning/restore/restore.go index 0ea46ea67bf14..63b76b45a11d0 100644 --- a/br/pkg/lightning/restore/restore.go +++ b/br/pkg/lightning/restore/restore.go @@ -1926,8 +1926,7 @@ func (rc *Controller) preCheckRequirements(ctx context.Context) error { if !taskExist && rc.taskMgr != nil { rc.taskMgr.CleanupTask(ctx) } - return errors.Errorf("tidb-lightning check failed."+ - " Please fix the failed check(s):\n %s", rc.checkTemplate.FailedMsg()) + return errors.Errorf("tidb-lightning pre-check failed: %s", rc.checkTemplate.FailedMsg()) } return nil } From 8a684164ce4e81fca1182764c13f9706eee93a9d Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 21 Dec 2021 17:26:09 +0800 Subject: [PATCH 2/3] remove useless test --- br/pkg/lightning/restore/check_info_test.go | 376 -------------------- 1 file changed, 376 deletions(-) diff --git a/br/pkg/lightning/restore/check_info_test.go b/br/pkg/lightning/restore/check_info_test.go index ccc4aa74c0c28..e5bb489cf72c6 100644 --- a/br/pkg/lightning/restore/check_info_test.go +++ b/br/pkg/lightning/restore/check_info_test.go @@ -16,394 +16,18 @@ package restore import ( "context" - "fmt" - "os" - "path/filepath" - . "github.com/pingcap/check" "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/br/pkg/lightning/checkpoints" "github.com/pingcap/tidb/br/pkg/lightning/config" - "github.com/pingcap/tidb/br/pkg/lightning/mydump" "github.com/pingcap/tidb/br/pkg/lightning/worker" "github.com/pingcap/tidb/br/pkg/storage" - "github.com/pingcap/tidb/ddl" - "github.com/pingcap/tidb/parser" - "github.com/pingcap/tidb/parser/ast" - "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - tmock "github.com/pingcap/tidb/util/mock" ) var _ = Suite(&checkInfoSuite{}) type checkInfoSuite struct{} -const passed CheckType = "pass" - -func (s *checkInfoSuite) TestCheckCSVHeader(c *C) { - dir := c.MkDir() - ctx := context.Background() - mockStore, err := storage.NewLocalStorage(dir) - c.Assert(err, IsNil) - - type tableSource struct { - Name string - SQL string - Sources []string - } - - cases := []struct { - ignoreColumns []*config.IgnoreColumns - // empty msg means check pass - level CheckType - Sources map[string][]*tableSource - }{ - - { - nil, - - passed, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8))", - []string{ - "aa,b\r\n", - }, - }, - }, - }, - }, - { - nil, - - passed, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8))", - []string{ - "a,b\r\ntest1,test2\r\n", - "aa,b\r\n", - }, - }, - }, - }, - }, - { - nil, - - Warn, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8))", - []string{ - "a,b\r\n", - }, - }, - }, - }, - }, - { - nil, - - Warn, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8))", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\n", - }, - }, - }, - }, - }, - { - nil, - - Warn, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`))", - []string{ - "a,b\r\ntest1,test2\r\n", - }, - }, - }, - }, - }, - { - nil, - - Critical, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`))", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\r\n", - }, - }, - }, - }, - }, - // ignore primary key, should still be warn - { - []*config.IgnoreColumns{ - { - DB: "db", - Table: "tbl1", - Columns: []string{"a"}, - }, - }, - Warn, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`))", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\r\n", - }, - }, - }, - }, - }, - // ignore primary key, but has other unique key - { - []*config.IgnoreColumns{ - { - DB: "db", - Table: "tbl1", - Columns: []string{"a"}, - }, - }, - Critical, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`), unique key uk (`b`))", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\r\n", - }, - }, - }, - }, - }, - // ignore primary key, non other unique key - { - []*config.IgnoreColumns{ - { - DB: "db", - Table: "tbl1", - Columns: []string{"a"}, - }, - }, - Warn, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(16), b varchar(8), PRIMARY KEY (`a`), KEY idx_b (`b`))", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\r\n", - }, - }, - }, - }, - }, - // non unique key, but data type inconsistent - { - nil, - Critical, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a bigint, b varchar(8));", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\r\n", - }, - }, - }, - }, - }, - // non unique key, but ignore inconsistent field - { - []*config.IgnoreColumns{ - { - DB: "db", - Table: "tbl1", - Columns: []string{"a"}, - }, - }, - Warn, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a bigint, b varchar(8));", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\r\n", - }, - }, - }, - }, - }, - // multiple tables, test the choose priority - { - nil, - Critical, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(8), b varchar(8));", - []string{ - "a,b\r\ntest1,test2\r\n", - }, - }, - { - "tbl2", - "create table tbl1 (a varchar(8) primary key, b varchar(8));", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\r\n", - }, - }, - }, - }, - }, - { - nil, - Critical, - map[string][]*tableSource{ - "db": { - { - "tbl1", - "create table tbl1 (a varchar(8), b varchar(8));", - []string{ - "a,b\r\ntest1,test2\r\n", - }, - }, - }, - "db2": { - { - "tbl2", - "create table tbl1 (a bigint, b varchar(8));", - []string{ - "a,b\r\ntest1,test2\r\n", - "a,b\r\ntest3,test4\r\n", - }, - }, - }, - }, - }, - } - - cfg := &config.Config{ - Mydumper: config.MydumperRuntime{ - ReadBlockSize: config.ReadBlockSize, - CSV: config.CSVConfig{ - Separator: ",", - Delimiter: `"`, - Header: false, - NotNull: false, - Null: `\N`, - BackslashEscape: true, - TrimLastSep: false, - }, - }, - } - rc := &Controller{ - cfg: cfg, - store: mockStore, - ioWorkers: worker.NewPool(context.Background(), 1, "io"), - } - - p := parser.New() - p.SetSQLMode(mysql.ModeANSIQuotes) - se := tmock.NewContext() - - for _, ca := range cases { - rc.checkTemplate = NewSimpleTemplate() - cfg.Mydumper.IgnoreColumns = ca.ignoreColumns - rc.dbInfos = make(map[string]*checkpoints.TidbDBInfo) - - dbMetas := make([]*mydump.MDDatabaseMeta, 0) - for db, tbls := range ca.Sources { - tblMetas := make([]*mydump.MDTableMeta, 0, len(tbls)) - dbInfo := &checkpoints.TidbDBInfo{ - Name: db, - Tables: make(map[string]*checkpoints.TidbTableInfo), - } - rc.dbInfos[db] = dbInfo - - for _, tbl := range tbls { - node, err := p.ParseOneStmt(tbl.SQL, "", "") - c.Assert(err, IsNil) - core, err := ddl.MockTableInfo(se, node.(*ast.CreateTableStmt), 0xabcdef) - c.Assert(err, IsNil) - core.State = model.StatePublic - dbInfo.Tables[tbl.Name] = &checkpoints.TidbTableInfo{ - ID: core.ID, - DB: db, - Name: tbl.Name, - Core: core, - } - - fileInfos := make([]mydump.FileInfo, 0, len(tbl.Sources)) - for i, s := range tbl.Sources { - fileName := fmt.Sprintf("%s.%s.%d.csv", db, tbl.Name, i) - err = os.WriteFile(filepath.Join(dir, fileName), []byte(s), 0o644) - c.Assert(err, IsNil) - fileInfos = append(fileInfos, mydump.FileInfo{ - FileMeta: mydump.SourceFileMeta{ - Path: fileName, - Type: mydump.SourceTypeCSV, - FileSize: int64(len(s)), - }, - }) - } - tblMetas = append(tblMetas, &mydump.MDTableMeta{ - DB: db, - Name: tbl.Name, - DataFiles: fileInfos, - }) - } - dbMetas = append(dbMetas, &mydump.MDDatabaseMeta{ - Name: db, - Tables: tblMetas, - }) - } - - err := rc.checkCSVHeader(ctx, dbMetas) - c.Assert(err, IsNil) - if ca.level != passed { - c.Assert(rc.checkTemplate.FailedCount(ca.level), Equals, 1) - } - } -} - func (s *checkInfoSuite) TestLocalResource(c *C) { dir := c.MkDir() mockStore, err := storage.NewLocalStorage(dir) From 61fb532104f89cce8dc6771f222c9f874ed3791b Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 14 Feb 2022 14:40:05 +0800 Subject: [PATCH 3/3] fmt code --- br/pkg/lightning/restore/check_info_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/br/pkg/lightning/restore/check_info_test.go b/br/pkg/lightning/restore/check_info_test.go index e5bb489cf72c6..98556d6f78ef7 100644 --- a/br/pkg/lightning/restore/check_info_test.go +++ b/br/pkg/lightning/restore/check_info_test.go @@ -16,6 +16,7 @@ package restore import ( "context" + . "github.com/pingcap/check" "github.com/pingcap/failpoint"