Merge release-3.0 to release-3.1

bindinfo/bind_test.go

@@ -76,6 +76,7 @@ func (s *testSuite) SetUpSuite(c *C) {
 		session.SetSchemaLease(0)
 		session.DisableStats4Test()
 	}
+	bindinfo.Lease = 0
 	d, err := session.BootstrapSession(s.store)
 	c.Assert(err, IsNil)
 	d.SetStatsUpdating(true)
@@ -99,8 +100,8 @@ func (s *testSuite) TearDownTest(c *C) {
 }
 
 func (s *testSuite) cleanBindingEnv(tk *testkit.TestKit) {
-	tk.MustExec("drop table if exists mysql.bind_info")
-	tk.MustExec(session.CreateBindInfoTable)
+	tk.MustExec("truncate table mysql.bind_info")
+	s.domain.BindHandle().Clear()
 }
 
 func (s *testSuite) TestBindParse(c *C) {
@@ -166,7 +167,7 @@ func (s *testSuite) TestGlobalBinding(c *C) {
 	metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb)
 	c.Assert(pb.GetGauge().GetValue(), Equals, float64(161))
 
-	sql, hash := parser.NormalizeDigest("select * from t where i > ?")
+	sql, hash := parser.NormalizeDigest("select * from t where i          >      30.0")
 
 	bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test")
 	c.Check(bindData, NotNil)
@@ -388,7 +389,7 @@ func (s *testSuite) TestExplain(c *C) {
 	tk.MustExec("create table t1(id int)")
 	tk.MustExec("create table t2(id int)")
 
-	tk.MustQuery("explain SELECT * from t1,t2 where t1.id = t2.id").Check(testkit.Rows(
+	tk.MustQuery("explain SELECT * from t1,t2 where t1.id = t2.id;").Check(testkit.Rows(
 		"HashLeftJoin_8 12487.50 root inner join, inner:TableReader_15, equal:[eq(test.t1.id, test.t2.id)]",
 		"├─TableReader_12 9990.00 root data:Selection_11",
 		"│ └─Selection_11 9990.00 cop not(isnull(test.t1.id))",
@@ -398,7 +399,7 @@ func (s *testSuite) TestExplain(c *C) {
 		"    └─TableScan_13 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo",
 	))
 
-	tk.MustQuery("explain SELECT  /*+ TIDB_SMJ(t1, t2) */  * from t1,t2 where t1.id = t2.id").Check(testkit.Rows(
+	tk.MustQuery("explain SELECT  /*+ TIDB_SMJ(t1, t2) */  * from t1,t2 where t1.id = t2.id;").Check(testkit.Rows(
 		"MergeJoin_7 12487.50 root inner join, left key:test.t1.id, right key:test.t2.id",
 		"├─Sort_11 9990.00 root test.t1.id:asc",
 		"│ └─TableReader_10 9990.00 root data:Selection_9",
@@ -412,7 +413,7 @@ func (s *testSuite) TestExplain(c *C) {
 
 	tk.MustExec("create global binding for SELECT * from t1,t2 where t1.id = t2.id using SELECT  /*+ TIDB_SMJ(t1, t2) */  * from t1,t2 where t1.id = t2.id")
 
-	tk.MustQuery("explain SELECT * from t1,t2 where t1.id = t2.id").Check(testkit.Rows(
+	tk.MustQuery("explain SELECT * from t1,t2 where t1.id = t2.id;").Check(testkit.Rows(
 		"MergeJoin_7 12487.50 root inner join, left key:test.t1.id, right key:test.t2.id",
 		"├─Sort_11 9990.00 root test.t1.id:asc",
 		"│ └─TableReader_10 9990.00 root data:Selection_9",
@@ -427,6 +428,42 @@ func (s *testSuite) TestExplain(c *C) {
 	tk.MustExec("drop global binding for SELECT * from t1,t2 where t1.id = t2.id")
 }
 
+// TestBindingSymbolList tests sql with "?, ?, ?, ?", fixes #13871
+func (s *testSuite) TestBindingSymbolList(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	s.cleanBindingEnv(tk)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b int, INDEX ia (a), INDEX ib (b));")
+	tk.MustExec("insert into t value(1, 1);")
+
+	// before binding
+	tk.MustQuery("select a, b from t where a = 3 limit 1, 100")
+	c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:ia")
+	c.Assert(tk.MustUseIndex("select a, b from t where a = 3 limit 1, 100", "a"), IsTrue)
+
+	tk.MustExec(`create global binding for select a, b from t where a = 1 limit 0, 1 using select a, b from t use index (ib) where a = 1 limit 0, 1`)
+
+	// after binding
+	tk.MustQuery("select a, b from t where a = 3 limit 1, 100;")
+	c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:ib")
+	c.Assert(tk.MustUseIndex("select a, b from t where a = 3 limit 1, 100", "b"), IsTrue)
+
+	// Normalize
+	sql, hash := parser.NormalizeDigest("select a, b from t where a = 1 limit 0, 1")
+
+	bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test")
+	c.Assert(bindData, NotNil)
+	c.Check(bindData.OriginalSQL, Equals, "select a , b from t where a = ? limit ...")
+	c.Check(bindData.BindSQL, Equals, "select a, b from t use index (ib) where a = 1 limit 0, 1")
+	c.Check(bindData.Db, Equals, "test")
+	c.Check(bindData.Status, Equals, "using")
+	c.Check(bindData.Charset, NotNil)
+	c.Check(bindData.Collation, NotNil)
+	c.Check(bindData.CreateTime, NotNil)
+	c.Check(bindData.UpdateTime, NotNil)
+}
+
 func (s *testSuite) TestErrorBind(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	s.cleanBindingEnv(tk)
@@ -465,3 +502,21 @@ func (s *testSuite) TestErrorBind(c *C) {
 	c.Check(err, IsNil)
 	c.Check(chk.NumRows(), Equals, 0)
 }
+
+func (s *testSuite) TestBindingCache(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	s.cleanBindingEnv(tk)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b int, index idx(a))")
+	tk.MustExec("create global binding for select * from t using select * from t use index(idx);")
+	tk.MustExec("create database tmp")
+	tk.MustExec("use tmp")
+	tk.MustExec("create table t(a int, b int, index idx(a))")
+	tk.MustExec("create global binding for select * from t using select * from t use index(idx);")
+
+	c.Assert(s.domain.BindHandle().Update(false), IsNil)
+	c.Assert(s.domain.BindHandle().Update(false), IsNil)
+	res := tk.MustQuery("show global bindings")
+	c.Assert(len(res.Rows()), Equals, 2)
+}

bindinfo/handle.go

@@ -16,7 +16,6 @@ package bindinfo
 import (
 	"context"
 	"fmt"
-	"go.uber.org/zap"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -31,6 +30,7 @@ import (
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/logutil"
 	"github.com/pingcap/tidb/util/sqlexec"
+	"go.uber.org/zap"
 )
 
 // BindHandle is used to handle all global sql bind operations.
@@ -95,7 +95,7 @@ func NewBindHandle(ctx sessionctx.Context) *BindHandle {
 func (h *BindHandle) Update(fullLoad bool) (err error) {
 	sql := "select original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation from mysql.bind_info"
 	if !fullLoad {
-		sql += " where update_time >= \"" + h.lastUpdateTime.String() + "\""
+		sql += " where update_time > \"" + h.lastUpdateTime.String() + "\""
 	}
 
 	// No need to acquire the session context lock for ExecRestrictedSQL, it
@@ -304,7 +304,7 @@ func (h *BindHandle) GetAllBindRecord() (bindRecords []*BindMeta) {
 }
 
 func (h *BindHandle) newBindMeta(record *BindRecord) (hash string, meta *BindMeta, err error) {
-	hash = parser.DigestHash(record.OriginalSQL)
+	hash = parser.DigestNormalized(record.OriginalSQL)
 	stmtNodes, _, err := h.bindInfo.parser.Parse(record.BindSQL, record.Charset, record.Collation)
 	if err != nil {
 		return "", nil, err
@@ -314,7 +314,7 @@ func (h *BindHandle) newBindMeta(record *BindRecord) (hash string, meta *BindMet
 }
 
 func newBindMetaWithoutAst(record *BindRecord) (hash string, meta *BindMeta) {
-	hash = parser.DigestHash(record.OriginalSQL)
+	hash = parser.DigestNormalized(record.OriginalSQL)
 	meta = &BindMeta{BindRecord: record}
 	return hash, meta
 }
@@ -358,6 +358,7 @@ func (c cache) removeDeletedBindMeta(hash string, meta *BindMeta, scope string)
 			}
 		}
 	}
+	c[hash] = metas
 }
 
 // removeStaleBindMetas removes all the stale BindMeta in the cache.
@@ -377,12 +378,15 @@ func (c cache) removeStaleBindMetas(hash string, meta *BindMeta, scope string) {
 			}
 		}
 	}
+	c[hash] = metas
 }
 
 func (c cache) copy() cache {
 	newCache := make(cache, len(c))
 	for k, v := range c {
-		newCache[k] = v
+		bindMetas := make([]*BindMeta, len(v))
+		copy(bindMetas, v)
+		newCache[k] = bindMetas
 	}
 	return newCache
 }
@@ -445,3 +449,10 @@ func (h *BindHandle) logicalDeleteBindInfoSQL(normdOrigSQL, db string, updateTs
 		expression.Quote(normdOrigSQL),
 		expression.Quote(db))
 }
+
+// Clear resets the bind handle. It is used for test.
+func (h *BindHandle) Clear() {
+	h.bindInfo.Store(make(cache))
+	h.invalidBindRecordMap.Store(make(map[string]*invalidBindRecordMap))
+	h.lastUpdateTime = types.ZeroTimestamp
+}

bindinfo/session_handle.go

@@ -45,7 +45,7 @@ func (h *SessionHandle) appendBindMeta(hash string, meta *BindMeta) {
 }
 
 func (h *SessionHandle) newBindMeta(record *BindRecord) (hash string, meta *BindMeta, err error) {
-	hash = parser.DigestHash(record.OriginalSQL)
+	hash = parser.DigestNormalized(record.OriginalSQL)
 	stmtNodes, _, err := h.parser.Parse(record.BindSQL, record.Charset, record.Collation)
 	if err != nil {
 		return "", nil, err
@@ -75,14 +75,14 @@ func (h *SessionHandle) AddBindRecord(record *BindRecord) error {
 func (h *SessionHandle) DropBindRecord(record *BindRecord) {
 	meta := &BindMeta{BindRecord: record}
 	meta.Status = deleted
-	hash := parser.DigestHash(record.OriginalSQL)
+	hash := parser.DigestNormalized(record.OriginalSQL)
 	h.ch.removeDeletedBindMeta(hash, meta, metrics.ScopeSession)
 	h.appendBindMeta(hash, meta)
 }
 
 // GetBindRecord return the BindMeta of the (normdOrigSQL,db) if BindMeta exist.
 func (h *SessionHandle) GetBindRecord(normdOrigSQL, db string) *BindMeta {
-	hash := parser.DigestHash(normdOrigSQL)
+	hash := parser.DigestNormalized(normdOrigSQL)
 	bindRecords := h.ch[hash]
 	if bindRecords != nil {
 		for _, bindRecord := range bindRecords {

cmd/explaintest/r/explain_easy.result

@@ -489,6 +489,37 @@ explain select distinct t1.a, t1.b from t1 left outer join t2 on t1.a = t2.a;
 id	count	task	operator info
 IndexReader_9	10000.00	root	index:IndexScan_8
 └─IndexScan_8	10000.00	cop	table:t1, index:a, b, range:[NULL,+inf], keep order:false, stats:pseudo
+CREATE TABLE `test01` (
+`id` bigint(20) NOT NULL AUTO_INCREMENT,
+`stat_date` int(11) NOT NULL DEFAULT '0',
+`show_date` varchar(20) NOT NULL DEFAULT '',
+`region_id` bigint(20) unsigned NOT NULL DEFAULT '0',
+`period` tinyint(3) unsigned NOT NULL DEFAULT '0',
+`registration_num` bigint(20) unsigned NOT NULL DEFAULT '0',
+PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+CREATE TABLE `test02` (
+`id` bigint(20) NOT NULL AUTO_INCREMENT,
+`region_name` varchar(128) DEFAULT NULL,
+PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+EXPLAIN SELECT COUNT(1) FROM (SELECT COALESCE(b.region_name, '不详') region_name, SUM(a.registration_num) registration_num FROM (SELECT stat_date, show_date, region_id, 0 registration_num FROM test01 WHERE period = 1 AND stat_date >= 20191202 AND stat_date <= 20191202 UNION ALL SELECT stat_date, show_date, region_id, registration_num registration_num FROM test01 WHERE period = 1 AND stat_date >= 20191202 AND stat_date <= 20191202) a LEFT JOIN test02 b ON a.region_id = b.id WHERE registration_num > 0 AND a.stat_date >= '20191202' AND a.stat_date <= '20191202' GROUP BY a.stat_date , a.show_date , COALESCE(b.region_name, '不详') ) JLS;
+id	count	task	operator info
+StreamAgg_22	1.00	root	funcs:count(1)
+└─HashAgg_25	1.00	root	group by:col_0, col_1, col_2, funcs:firstrow(1)
+  └─Projection_41	0.01	root	a.stat_date, a.show_date, coalesce(test.b.region_name, "不详")
+    └─IndexJoin_28	0.01	root	left outer join, inner:TableReader_27, outer key:a.region_id, inner key:test.b.id
+      ├─Union_30	0.01	root	
+      │ ├─Projection_31	0.00	root	test.test01.stat_date, test.test01.show_date, test.test01.region_id, cast(registration_num)
+      │ │ └─Projection_32	0.00	root	test.test01.stat_date, test.test01.show_date, test.test01.region_id, 0
+      │ │   └─TableDual_33	0.00	root	rows:0
+      │ └─Projection_34	0.01	root	test.test01.stat_date, test.test01.show_date, test.test01.region_id, cast(test.test01.registration_num)
+      │   └─Selection_35	0.01	root	gt(cast(test.test01.registration_num), 0)
+      │     └─TableReader_38	0.01	root	data:Selection_37
+      │       └─Selection_37	0.01	cop	eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), ge(test.test01.stat_date, 20191202), le(test.test01.stat_date, 20191202), le(test.test01.stat_date, 20191202)
+      │         └─TableScan_36	10000.00	cop	table:test01, range:[-inf,+inf], keep order:false, stats:pseudo
+      └─TableReader_27	1.00	root	data:TableScan_26
+        └─TableScan_26	1.00	cop	table:b, range: decided by [a.region_id], keep order:false, stats:pseudo
 drop table if exists t;
 create table t(a int, nb int not null, nc int not null);
 explain select ifnull(a, 0) from t;

cmd/explaintest/r/explain_union_scan.result

@@ -0,0 +1,32 @@
+drop table if exists city;
+CREATE TABLE `city` (
+`id` varchar(70) NOT NULL,
+`province_id` int(15) DEFAULT NULL,
+`city_name` varchar(90) DEFAULT NULL,
+`description` varchar(90) DEFAULT NULL,
+PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
+load stats "s/explain_union_scan.json";
+insert into city values("06766b3ef41d484d8878606393f1ed0b", 88, "chongqing", "chongqing city");
+begin;
+update city set province_id = 77 where id="06766b3ef41d484d8878606393f1ed0b";
+explain select t1.*, t2.province_id as provinceID, t2.city_name as cityName, t3.description as description from city t1 inner join city t2 on t1.id = t2.id left join city t3 on t1.province_id = t3.province_id where t1.province_id > 1 and t1.province_id < 100 limit 10;
+id	count	task	operator info
+Projection_17	10.00	root	test.t1.id, test.t1.province_id, test.t1.city_name, test.t1.description, test.t2.province_id, test.t2.city_name, test.t3.description
+└─Limit_20	10.00	root	offset:0, count:10
+  └─HashLeftJoin_21	10.00	root	left outer join, inner:UnionScan_52, equal:[eq(test.t1.province_id, test.t3.province_id)]
+    ├─Limit_24	10.00	root	offset:0, count:10
+    │ └─IndexJoin_29	10.00	root	inner join, inner:UnionScan_28, outer key:test.t1.id, inner key:test.t2.id
+    │   ├─UnionScan_38	10.00	root	gt(test.t1.province_id, 1), lt(test.t1.province_id, 100)
+    │   │ └─TableReader_41	10.00	root	data:Selection_40
+    │   │   └─Selection_40	10.00	cop	gt(test.t1.province_id, 1), lt(test.t1.province_id, 100)
+    │   │     └─TableScan_39	14.30	cop	table:t1, range:[-inf,+inf], keep order:false
+    │   └─UnionScan_28	1.00	root	
+    │     └─IndexLookUp_27	1.00	root	
+    │       ├─IndexScan_25	1.00	cop	table:t2, index:id, range: decided by [eq(test.t2.id, test.t1.id)], keep order:false
+    │       └─TableScan_26	1.00	cop	table:t2, keep order:false
+    └─UnionScan_52	519304.44	root	gt(test.t3.province_id, 1), lt(test.t3.province_id, 100), not(isnull(test.t3.province_id))
+      └─TableReader_55	519304.44	root	data:Selection_54
+        └─Selection_54	519304.44	cop	gt(test.t3.province_id, 1), lt(test.t3.province_id, 100), not(isnull(test.t3.province_id))
+          └─TableScan_53	536284.00	cop	table:t3, range:[-inf,+inf], keep order:false
+commit;