From cbae3d6b5b08c40889dab14fd42f448676261403 Mon Sep 17 00:00:00 2001 From: ciscoxll Date: Wed, 16 Jan 2019 19:46:11 +0800 Subject: [PATCH] ddl: modify different character sets varchar type column maximum length limit (#9050) --- ddl/db_test.go | 24 +++++++++++++++++ ddl/ddl_api.go | 53 ++++++++++++++++++++++++++++++++++++++ planner/core/preprocess.go | 17 ++++-------- 3 files changed, 82 insertions(+), 12 deletions(-) diff --git a/ddl/db_test.go b/ddl/db_test.go index 8d23bbc57b598..d8b9f7c68ca9b 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -3848,6 +3848,30 @@ func (s *testDBSuite) TestTransactionWithWriteOnlyColumn(c *C) { s.tk.MustQuery("select a from t1").Check(testkit.Rows("2")) } +func (s *testDBSuite) TestCheckTooBigFieldLength(c *C) { + s.tk = testkit.NewTestKit(c, s.store) + s.tk.MustExec("use test") + s.tk.MustExec("drop table if exists tr_01;") + s.tk.MustExec("create table tr_01 (id int, name varchar(20000), purchased date ) default charset=utf8 collate=utf8_bin;") + + s.tk.MustExec("drop table if exists tr_02;") + s.tk.MustExec("create table tr_02 (id int, name varchar(16000), purchased date ) default charset=utf8mb4 collate=utf8mb4_bin;") + + s.tk.MustExec("drop table if exists tr_03;") + s.tk.MustExec("create table tr_03 (id int, name varchar(65534), purchased date ) default charset=latin1;") + + s.tk.MustExec("drop table if exists tr_03;") + s.tk.MustExec("create table tr_03 (a varchar(16000) ) default charset utf8;") + s.tk.MustExec("alter table tr_03 convert to character set utf8mb4;") + + s.tk.MustExec("drop table if exists tr_04;") + s.tk.MustExec("create table tr_04 (a varchar(20000) ) default charset utf8;") + s.testErrorCode(c, "alter table tr_04 convert to character set utf8mb4;", tmysql.ErrTooBigFieldlength) + s.testErrorCode(c, "create table tr_05 (id int, name varchar(30000), purchased date ) default charset=utf8 collate=utf8_bin;", tmysql.ErrTooBigFieldlength) + s.testErrorCode(c, "create table tr_05 (id int, name varchar(20000) charset utf8mb4, purchased date ) default charset=utf8 collate=utf8;", tmysql.ErrTooBigFieldlength) + s.testErrorCode(c, "create table tr_05 (id int, name varchar(65536), purchased date ) default charset=latin1;", tmysql.ErrTooBigFieldlength) +} + func (s *testDBSuite) TestAddColumn2(c *C) { s.tk = testkit.NewTestKit(c, s.store) s.mustExec(c, "use test_db") diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 6f2bcd1c8be2d..820bd7ef8aa78 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -613,6 +613,42 @@ func checkColumnsAttributes(colDefs []*ast.ColumnDef) error { return nil } +// checkColumnFieldLength check the maximum length limit for different character set varchar type columns. +func checkColumnFieldLength(schema *model.DBInfo, colDefs []*ast.ColumnDef, tbInfo *model.TableInfo) error { + for _, colDef := range colDefs { + if colDef.Tp.Tp == mysql.TypeVarchar { + var setCharset string + setCharset = mysql.DefaultCharset + if len(schema.Charset) != 0 { + setCharset = schema.Charset + } + if len(tbInfo.Charset) != 0 { + setCharset = tbInfo.Charset + } + + err := IsTooBigFieldLength(colDef.Tp.Flen, colDef.Name.Name.O, setCharset) + if err != nil { + return errors.Trace(err) + } + } + } + return nil +} + +// IsTooBigFieldLength check if the varchar type column exceeds the maximum length limit. +func IsTooBigFieldLength(colDefTpFlen int, colDefName, setCharset string) error { + desc, err := charset.GetCharsetDesc(setCharset) + if err != nil { + return errors.Trace(err) + } + maxFlen := mysql.MaxFieldVarCharLength + maxFlen /= desc.Maxlen + if colDefTpFlen != types.UnspecifiedLength && colDefTpFlen > maxFlen { + return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDefName, maxFlen) + } + return nil +} + // checkColumnAttributes check attributes for single column. func checkColumnAttributes(colName string, tp *types.FieldType) error { switch tp.Tp { @@ -955,6 +991,10 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e if err != nil { return errors.Trace(err) } + if err = checkColumnFieldLength(schema, s.Cols, tbInfo); err != nil { + return errors.Trace(err) + } + err = d.doDDLJob(ctx, job) if err == nil { if tbInfo.AutoIncID > 1 { @@ -1722,6 +1762,11 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or if !mysql.HasNotNullFlag(col.Flag) && mysql.HasNotNullFlag(newCol.Flag) { return nil, errUnsupportedModifyColumn.GenWithStackByArgs("null to not null") } + + if err = checkColumnFieldLength(schema, spec.NewColumns, t.Meta()); err != nil { + return nil, errors.Trace(err) + } + // As same with MySQL, we don't support modifying the stored status for generated columns. if err = checkModifyGeneratedColumn(t.Cols(), col, newCol); err != nil { return nil, errors.Trace(err) @@ -1902,6 +1947,14 @@ func (d *ddl) AlterTableCharsetAndCollate(ctx sessionctx.Context, ident ast.Iden return errors.Trace(err) } + for _, col := range tb.Meta().Cols() { + if col.Tp == mysql.TypeVarchar { + if err = IsTooBigFieldLength(col.Flen, col.Name.O, toCharset); err != nil { + return errors.Trace(err) + } + } + } + job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index 115442cad9dc1..350ec424b75ec 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl" @@ -469,22 +468,16 @@ func checkColumn(colDef *ast.ColumnDef) error { return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDef.Name.Name.O, mysql.MaxFieldCharLength) } case mysql.TypeVarchar: - maxFlen := mysql.MaxFieldVarCharLength - cs := tp.Charset - // TODO: TableDefaultCharset-->DatabaseDefaultCharset-->SystemDefaultCharset. - // TODO: Change TableOption parser to parse collate. - // Reference https://github.com/pingcap/tidb/blob/b091e828cfa1d506b014345fb8337e424a4ab905/ddl/ddl_api.go#L185-L204 if len(tp.Charset) == 0 { - cs = mysql.DefaultCharset + // It's not easy to get the schema charset and table charset here. + // The charset is determined by the order ColumnDefaultCharset --> TableDefaultCharset-->DatabaseDefaultCharset-->SystemDefaultCharset. + // return nil, to make the check in the ddl.CreateTable. + return nil } - desc, err := charset.GetCharsetDesc(cs) + err := ddl.IsTooBigFieldLength(colDef.Tp.Flen, colDef.Name.Name.O, tp.Charset) if err != nil { return errors.Trace(err) } - maxFlen /= desc.Maxlen - if tp.Flen != types.UnspecifiedLength && tp.Flen > maxFlen { - return types.ErrTooBigFieldLength.GenWithStack("Column length too big for column '%s' (max = %d); use BLOB or TEXT instead", colDef.Name.Name.O, maxFlen) - } case mysql.TypeFloat, mysql.TypeDouble: if tp.Decimal > mysql.MaxFloatingTypeScale { return types.ErrTooBigScale.GenWithStackByArgs(tp.Decimal, colDef.Name.Name.O, mysql.MaxFloatingTypeScale)