planner: make unsafe substitute can be used in generated column (#37779)

close #35490
pingcap · Sep 16, 2022 · 20d46c2 · 20d46c2
1 parent 77c8b6b
commit 20d46c2
Show file tree

Hide file tree

Showing 7 changed files with 151 additions and 1 deletion.
diff --git a/cmd/explaintest/r/explain_generate_column_substitute.result b/cmd/explaintest/r/explain_generate_column_substitute.result
@@ -642,3 +642,87 @@ HashJoin	9.99	root		inner join, equal:[eq(test.t2.c_decimal, test.t1.c_decimal)]
     └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 drop table t1;
 drop table t2;
+set @@tidb_enable_unsafe_substitute=1;
+CREATE TABLE person (id INT PRIMARY KEY,address_info JSON,city VARCHAR(2) AS (JSON_UNQUOTE(address_info->"$.city")),KEY (city));
+INSERT INTO `person` (`id`, `address_info`) VALUES('1','{\"city\": \"Beijing\"}');
+SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+id
+1
+desc format = 'brief' SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+id	estRows	task	access object	operator info
+Projection	8000.00	root		test.person.id
+└─TableReader	8000.00	root		data:Selection
+  └─Selection	8000.00	cop[tikv]		eq(json_unquote(cast(json_extract(test.person.address_info, "$.city"), var_string(16777216))), "Beijing")
+    └─TableFullScan	10000.00	cop[tikv]	table:person	keep order:false, stats:pseudo
+SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+id
+desc format = 'brief' SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+id	estRows	task	access object	operator info
+Projection	10.00	root		test.person.id
+└─IndexReader	10.00	root		index:IndexRangeScan
+  └─IndexRangeScan	10.00	cop[tikv]	table:person, index:city(city)	range:["Beijing","Beijing"], keep order:false, stats:pseudo
+drop table person;
+create table t(a char(5), b char(6) as (concat(a, a)), index bx(b));
+insert into t(a) values ('aaaaa');
+select * from t;
+a	b
+aaaaa	aaaaaa
+select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa';
+a	b
+aaaaa	aaaaaa
+desc format = 'brief' select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa';
+id	estRows	task	access object	operator info
+TableReader	8000.00	root		data:Selection
+└─Selection	8000.00	cop[tikv]		eq(concat(test.t.a, test.t.a), "aaaaaaaaaa")
+  └─TableFullScan	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
+select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa';
+a	b
+desc format = 'brief' select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa';
+id	estRows	task	access object	operator info
+IndexLookUp	10.00	root		
+├─IndexRangeScan(Build)	10.00	cop[tikv]	table:t, index:bx(b)	range:["aaaaaaaaaa","aaaaaaaaaa"], keep order:false, stats:pseudo
+└─TableRowIDScan(Probe)	10.00	cop[tikv]	table:t	keep order:false, stats:pseudo
+drop table t;
+CREATE TABLE person (id INT PRIMARY KEY,address_info JSON,city VARCHAR(64) AS (JSON_UNQUOTE(address_info->"$.city")),KEY (city));
+INSERT INTO `person` (`id`, `address_info`) VALUES('1','{\"city\": \"Beijing\"}');
+SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+id
+1
+desc format = 'brief' SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+id	estRows	task	access object	operator info
+Projection	8000.00	root		test.person.id
+└─TableReader	8000.00	root		data:Selection
+  └─Selection	8000.00	cop[tikv]		eq(json_unquote(cast(json_extract(test.person.address_info, "$.city"), var_string(16777216))), "Beijing")
+    └─TableFullScan	10000.00	cop[tikv]	table:person	keep order:false, stats:pseudo
+SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+id
+1
+desc format = 'brief' SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+id	estRows	task	access object	operator info
+Projection	10.00	root		test.person.id
+└─IndexReader	10.00	root		index:IndexRangeScan
+  └─IndexRangeScan	10.00	cop[tikv]	table:person, index:city(city)	range:["Beijing","Beijing"], keep order:false, stats:pseudo
+drop table person;
+create table t(a char(5), b char(10) as (concat(a, a)), index bx(b));
+insert into t(a) values ('aaaaa');
+select * from t;
+a	b
+aaaaa	aaaaaaaaaa
+select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa';
+a	b
+aaaaa	aaaaaaaaaa
+desc format = 'brief' select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa';
+id	estRows	task	access object	operator info
+TableReader	8000.00	root		data:Selection
+└─Selection	8000.00	cop[tikv]		eq(concat(test.t.a, test.t.a), "aaaaaaaaaa")
+  └─TableFullScan	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
+select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa';
+a	b
+aaaaa	aaaaaaaaaa
+desc format = 'brief' select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa';
+id	estRows	task	access object	operator info
+IndexLookUp	10.00	root		
+├─IndexRangeScan(Build)	10.00	cop[tikv]	table:t, index:bx(b)	range:["aaaaaaaaaa","aaaaaaaaaa"], keep order:false, stats:pseudo
+└─TableRowIDScan(Probe)	10.00	cop[tikv]	table:t	keep order:false, stats:pseudo
+drop table t;
+set @@tidb_enable_unsafe_substitute=0;
diff --git a/cmd/explaintest/t/explain_generate_column_substitute.test b/cmd/explaintest/t/explain_generate_column_substitute.test
@@ -302,3 +302,39 @@ explain format = 'brief' select /*+ agg_to_cop() */ * from t1 where c_decimal in
 drop table t1;
 drop table t2;
 
+set @@tidb_enable_unsafe_substitute=1;
+CREATE TABLE person (id INT PRIMARY KEY,address_info JSON,city VARCHAR(2) AS (JSON_UNQUOTE(address_info->"$.city")),KEY (city));
+INSERT INTO `person` (`id`, `address_info`) VALUES('1','{\"city\": \"Beijing\"}');
+SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+desc format = 'brief' SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+desc format = 'brief' SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+drop table person;
+
+create table t(a char(5), b char(6) as (concat(a, a)), index bx(b));
+insert into t(a) values ('aaaaa');
+select * from t;
+select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa';
+desc format = 'brief' select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa';
+select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa';
+desc format = 'brief' select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa';
+drop table t;
+
+CREATE TABLE person (id INT PRIMARY KEY,address_info JSON,city VARCHAR(64) AS (JSON_UNQUOTE(address_info->"$.city")),KEY (city));
+INSERT INTO `person` (`id`, `address_info`) VALUES('1','{\"city\": \"Beijing\"}');
+SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+desc format = 'brief' SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+desc format = 'brief' SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing';
+drop table person;
+
+create table t(a char(5), b char(10) as (concat(a, a)), index bx(b));
+insert into t(a) values ('aaaaa');
+select * from t;
+select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa';
+desc format = 'brief' select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa';
+select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa';
+desc format = 'brief' select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa';
+drop table t;
+set @@tidb_enable_unsafe_substitute=0;
+
diff --git a/parser/types/field_type.go b/parser/types/field_type.go
@@ -273,6 +273,26 @@ func (ft *FieldType) Equal(other *FieldType) bool {
 	return true
 }
 
+// PartialEqual checks whether two FieldType objects are equal.
+// If unsafe is true and the objects is string type, PartialEqual will ignore flen.
+// See https://github.com/pingcap/tidb/issues/35490#issuecomment-1211658886 for more detail.
+func (ft *FieldType) PartialEqual(other *FieldType, unsafe bool) bool {
+	if !unsafe || ft.EvalType() != ETString || other.EvalType() != ETString {
+		return ft.Equal(other)
+	}
+
+	partialEqual := ft.charset == other.charset && ft.collate == other.collate && mysql.HasUnsignedFlag(ft.flag) == mysql.HasUnsignedFlag(other.flag)
+	if !partialEqual || len(ft.elems) != len(other.elems) {
+		return false
+	}
+	for i := range ft.elems {
+		if ft.elems[i] != other.elems[i] {
+			return false
+		}
+	}
+	return true
+}
+
 // EvalType gets the type in evaluation.
 func (ft *FieldType) EvalType() EvalType {
 	switch ft.tp {

diff --git a/planner/core/rule_generate_column_substitute.go b/planner/core/rule_generate_column_substitute.go
@@ -66,7 +66,7 @@ func collectGenerateColumn(lp LogicalPlan, exprToColumn ExprColumnMap) {
 			if colInfo.IsGenerated() && !colInfo.GeneratedStored {
 				s := ds.schema.Columns
 				col := expression.ColInfo2Col(s, colInfo)
-				if col != nil && col.GetType().Equal(col.VirtualExpr.GetType()) {
+				if col != nil && col.GetType().PartialEqual(col.VirtualExpr.GetType(), lp.SCtx().GetSessionVars().EnableUnsafeSubstitute) {
 					exprToColumn[col.VirtualExpr] = col
 				}
 			}

diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go
@@ -1263,6 +1263,9 @@ type SessionVars struct {
 	// EnableTiFlashReadForWriteStmt indicates whether to enable TiFlash to read for write statements.
 	EnableTiFlashReadForWriteStmt bool
 
+	// EnableUnsafeSubstitute indicates whether to enable generate column takes unsafe substitute.
+	EnableUnsafeSubstitute bool
+
 	// ForeignKeyChecks indicates whether to enable foreign key constraint check.
 	ForeignKeyChecks bool
 

diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go
@@ -1823,6 +1823,10 @@ var defaultSysVars = []*SysVar{
 		s.EnableTiFlashReadForWriteStmt = TiDBOptOn(val)
 		return nil
 	}},
+	{Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableUnsafeSubstitute, Value: BoolToOnOff(false), Type: TypeBool, SetSession: func(s *SessionVars, val string) error {
+		s.EnableUnsafeSubstitute = TiDBOptOn(val)
+		return nil
+	}},
 	{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptRangeMaxSize, Value: strconv.FormatInt(DefTiDBOptRangeMaxSize, 10), Type: TypeInt, MinValue: 0, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error {
 		s.RangeMaxSize = TidbOptInt64(val, DefTiDBOptRangeMaxSize)
 		return nil

diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go
@@ -238,6 +238,9 @@ const (
 	// TiFlashFastScan indicates whether use fast scan in tiflash.
 	TiFlashFastScan = "tiflash_fastscan"
 
+	// TiDBEnableUnsafeSubstitute indicates whether to enable generate column takes unsafe substitute.
+	TiDBEnableUnsafeSubstitute = "tidb_enable_unsafe_substitute"
+
 	// TiDBEnableTiFlashReadForWriteStmt indicates whether to enable TiFlash to read for write statements.
 	TiDBEnableTiFlashReadForWriteStmt = "tidb_enable_tiflash_read_for_write_stmt"
 )