Merge pull request #122301 from mgartner/backport23.1-120384

release-23.1: sql: add FORCE_INVERTED_INDEX hint

docs/generated/sql/bnf/stmt_block.bnf

@@ -1119,6 +1119,7 @@ unreserved_keyword ::=
 	| 'FORCE_NULL'
 	| 'FORCE_QUOTE'
 	| 'FORCE_INDEX'
+	| 'FORCE_INVERTED_INDEX'
 	| 'FORCE_ZIGZAG'
 	| 'FORWARD'
 	| 'FREEZE'
@@ -3624,6 +3625,7 @@ bare_label_keywords ::=
 	| 'FORCE_NULL'
 	| 'FORCE_QUOTE'
 	| 'FORCE_INDEX'
+	| 'FORCE_INVERTED_INDEX'
 	| 'FORCE_ZIGZAG'
 	| 'FOREIGN'
 	| 'FORMAT'

pkg/sql/opt/exec/execbuilder/relational.go

@@ -756,6 +756,10 @@ func (b *Builder) buildScan(scan *memo.ScanExpr) (execPlan, error) {
 		}
 	}
 
+	if scan.Flags.ForceInvertedIndex && !scan.IsInvertedScan() {
+		return execPlan{}, fmt.Errorf("could not produce a query plan conforming to the FORCE_INVERTED_INDEX hint")
+	}
+
 	idx := tab.Index(scan.Index)
 	if idx.IsInverted() && len(scan.InvertedConstraint) == 0 {
 		return execPlan{},

pkg/sql/opt/exec/execbuilder/testdata/inverted_index

@@ -224,6 +224,9 @@ Put /Table/108/1/0/0 -> /TUPLE/
 Del /Table/108/2/1/0/0
 InitPut /Table/108/2/15/0/0 -> /BYTES/
 
+statement error pgcode XXUUU could not produce a query plan conforming to the FORCE_INVERTED_INDEX hint
+SELECT * from d@{FORCE_INVERTED_INDEX}
+
 query T
 EXPLAIN (VERBOSE) SELECT * from d where b @>'{"a": "b"}'
 ----

pkg/sql/opt/memo/expr.go

@@ -424,10 +424,12 @@ type ScanFlags struct {
 
 	// ForceIndex forces the use of a specific index (specified in Index).
 	// ForceIndex and NoIndexJoin cannot both be set at the same time.
-	ForceIndex  bool
-	ForceZigzag bool
-	Direction   tree.Direction
-	Index       int
+	ForceIndex bool
+	// ForceInvertedIndex forces the use of an inverted index.
+	ForceInvertedIndex bool
+	ForceZigzag        bool
+	Direction          tree.Direction
+	Index              int
 
 	// When the optimizer is performing unique constraint or foreign key
 	// constraint check, we will temporarily disable the not visible index feature
@@ -753,6 +755,12 @@ func (s *ScanPrivate) IsFullIndexScan(md *opt.Metadata) bool {
 		s.HardLimit == 0
 }
 
+// IsInvertedScan returns true if the index being scanned is an inverted
+// index.
+func (s *ScanPrivate) IsInvertedScan() bool {
+	return s.InvertedConstraint != nil
+}
+
 // IsVirtualTable returns true if the table being scanned is a virtual table.
 func (s *ScanPrivate) IsVirtualTable(md *opt.Metadata) bool {
 	tab := md.Table(s.Table)

pkg/sql/opt/memo/interner.go

@@ -516,6 +516,7 @@ func (h *hasher) HashScanFlags(val ScanFlags) {
 	h.HashBool(val.NoZigzagJoin)
 	h.HashBool(val.NoFullScan)
 	h.HashBool(val.ForceIndex)
+	h.HashBool(val.ForceInvertedIndex)
 	h.HashBool(val.ForceZigzag)
 	h.HashInt(int(val.Direction))
 	h.HashUint64(uint64(val.Index))

pkg/sql/opt/memo/interner_test.go

@@ -378,7 +378,7 @@ func TestInterner(t *testing.T) {
 		{hashFn: in.hasher.HashScanFlags, eqFn: in.hasher.IsScanFlagsEqual, variations: []testVariation{
 			// Use unnamed fields so that compilation fails if a new field is
 			// added to ScanFlags.
-			{val1: ScanFlags{false, false, false, false, false, 0, 0, false, intsets.Fast{}}, val2: ScanFlags{}, equal: true},
+			{val1: ScanFlags{false, false, false, false, false, false, 0, 0, false, intsets.Fast{}}, val2: ScanFlags{}, equal: true},
 			{val1: ScanFlags{}, val2: ScanFlags{}, equal: true},
 			{val1: ScanFlags{NoIndexJoin: false}, val2: ScanFlags{NoIndexJoin: true}, equal: false},
 			{val1: ScanFlags{NoIndexJoin: true}, val2: ScanFlags{NoIndexJoin: true}, equal: true},
@@ -391,6 +391,10 @@ func TestInterner(t *testing.T) {
 			{val1: ScanFlags{NoIndexJoin: true, Index: 1}, val2: ScanFlags{NoIndexJoin: true, Index: 1}, equal: true},
 			{val1: ScanFlags{NoIndexJoin: true, Index: 1}, val2: ScanFlags{NoIndexJoin: true, Index: 2}, equal: false},
 			{val1: ScanFlags{NoIndexJoin: true, Index: 1}, val2: ScanFlags{NoIndexJoin: false, Index: 1}, equal: false},
+			{val1: ScanFlags{NoFullScan: true}, val2: ScanFlags{NoFullScan: false}, equal: false},
+			{val1: ScanFlags{NoFullScan: true}, val2: ScanFlags{NoFullScan: true}, equal: true},
+			{val1: ScanFlags{ForceInvertedIndex: true}, val2: ScanFlags{ForceInvertedIndex: false}, equal: false},
+			{val1: ScanFlags{ForceInvertedIndex: true}, val2: ScanFlags{ForceInvertedIndex: true}, equal: true},
 			{
 				val1:  ScanFlags{NoIndexJoin: true, ForceIndex: true, Direction: tree.Ascending, Index: 1},
 				val2:  ScanFlags{NoIndexJoin: false, ForceIndex: true, Direction: tree.Ascending, Index: 1},

pkg/sql/opt/optbuilder/select.go

@@ -624,6 +624,7 @@ func (b *Builder) buildScan(
 		private.Flags.NoIndexJoin = indexFlags.NoIndexJoin
 		private.Flags.NoZigzagJoin = indexFlags.NoZigzagJoin
 		private.Flags.NoFullScan = indexFlags.NoFullScan
+		private.Flags.ForceInvertedIndex = indexFlags.ForceInvertedIndex
 		if indexFlags.Index != "" || indexFlags.IndexID != 0 {
 			idx := -1
 			for i := 0; i < tab.IndexCount(); i++ {

pkg/sql/opt/xform/coster.go

@@ -759,6 +759,10 @@ func (c *coster) computeScanCost(scan *memo.ScanExpr, required *physical.Require
 		}
 	}
 
+	if scan.Flags.ForceInvertedIndex && !scan.IsInvertedScan() {
+		return hugeCost
+	}
+
 	stats := scan.Relational().Statistics()
 	rowCount := stats.RowCount
 	if isUnfiltered && c.evalCtx != nil && c.evalCtx.SessionData().DisallowFullTableScans {

pkg/sql/opt/xform/testdata/coster/scan

@@ -641,6 +641,79 @@ select
  └── filters
       └── c:4 > 0 [outer=(4), constraints=(/4: [/1 - ]; tight)]
 
+# Test that the FORCE_INVERTED_INDEX hint creates a huge cost for full table
+# scans.
+opt
+SELECT * FROM t@{FORCE_INVERTED_INDEX}
+----
+scan t
+ ├── columns: k:1!null a:2 b:3 c:4
+ ├── partial index predicates
+ │    └── b_partial: filters
+ │         └── c:4 > 0 [outer=(4), constraints=(/4: [/1 - ]; tight)]
+ ├── flags:
+ ├── stats: [rows=100000]
+ ├── cost: 1e+100
+ ├── key: (1)
+ └── fd: (1)-->(2-4)
+
+# Test that the FORCE_INVERTED_INDEX hint creates a huge cost for scans on
+# non-inverted indexes.
+opt
+SELECT * FROM t@{FORCE_INVERTED_INDEX} WHERE b = 0
+----
+select
+ ├── columns: k:1!null a:2 b:3!null c:4
+ ├── stats: [rows=2e-05, distinct(3)=2e-05, null(3)=0]
+ │   histogram(3)=  0  0
+ │                <--- 0
+ ├── cost: 1e+100
+ ├── key: (1)
+ ├── fd: ()-->(3), (1)-->(2,4)
+ ├── scan t
+ │    ├── columns: k:1!null a:2 b:3 c:4
+ │    ├── partial index predicates
+ │    │    └── b_partial: filters
+ │    │         └── c:4 > 0 [outer=(4), constraints=(/4: [/1 - ]; tight)]
+ │    ├── flags:
+ │    ├── stats: [rows=100000, distinct(1)=3, null(1)=0, distinct(3)=100, null(3)=0]
+ │    │   histogram(3)=  0  0  1e+05  0
+ │    │                <--- 0 ------- 10
+ │    ├── cost: 1e+100
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-4)
+ └── filters
+      └── b:3 = 0 [outer=(3), constraints=(/3: [/0 - /0]; tight), fd=()-->(3)]
+
+exec-ddl
+CREATE TABLE inv (
+  k INT PRIMARY KEY,
+  j JSON,
+  INVERTED INDEX (j)
+)
+----
+
+# Test that FORCE_INVERTED_INDEX does not create a huge cost for scans on
+# inverted indexes.
+opt
+SELECT * FROM inv@{FORCE_INVERTED_INDEX} WHERE j->'a' = '1'
+----
+index-join inv
+ ├── columns: k:1!null j:2
+ ├── immutable
+ ├── stats: [rows=111.1111]
+ ├── cost: 803.595556
+ ├── key: (1)
+ ├── fd: (1)-->(2)
+ └── scan inv@inv_j_idx
+      ├── columns: k:1!null
+      ├── inverted constraint: /5/1
+      │    └── spans: ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"]
+      ├── flags:
+      ├── stats: [rows=111.1111, distinct(5)=100, null(5)=0]
+      ├── cost: 132.464444
+      └── key: (1)
+
 exec-ddl
 CREATE TABLE b (x INT, y INT, z INT, s STRING, INDEX xs (x, s), INDEX xyz (x, y, z))
 ----

pkg/sql/parser/sql.y

@@ -917,7 +917,7 @@ func (u *sqlSymUnion) beginTransaction() *tree.BeginTransaction {
 
 %token <str> FAILURE FALSE FAMILY FETCH FETCHVAL FETCHTEXT FETCHVAL_PATH FETCHTEXT_PATH
 %token <str> FILES FILTER
-%token <str> FIRST FLOAT FLOAT4 FLOAT8 FLOORDIV FOLLOWING FOR FORCE FORCE_INDEX
+%token <str> FIRST FLOAT FLOAT4 FLOAT8 FLOORDIV FOLLOWING FOR FORCE FORCE_INDEX FORCE_INVERTED_INDEX
 %token <str> FORCE_NOT_NULL FORCE_NULL FORCE_QUOTE FORCE_ZIGZAG
 %token <str> FOREIGN FORMAT FORWARD FREEZE FROM FULL FUNCTION FUNCTIONS
 
@@ -12900,34 +12900,33 @@ index_flags_param:
   {
     $$.val = &tree.IndexFlags{NoIndexJoin: true}
   }
-|
-  NO_ZIGZAG_JOIN
+| NO_ZIGZAG_JOIN
   {
     $$.val = &tree.IndexFlags{NoZigzagJoin: true}
   }
-|
-  NO_FULL_SCAN
+| NO_FULL_SCAN
   {
     $$.val = &tree.IndexFlags{NoFullScan: true}
   }
-|
-  IGNORE_FOREIGN_KEYS
+| IGNORE_FOREIGN_KEYS
   {
     /* SKIP DOC */
     $$.val = &tree.IndexFlags{IgnoreForeignKeys: true}
   }
-|
-  FORCE_ZIGZAG
+| FORCE_INVERTED_INDEX
+  {
+    /* SKIP DOC */
+     $$.val = &tree.IndexFlags{ForceInvertedIndex: true}
+  }
+| FORCE_ZIGZAG
   {
      $$.val = &tree.IndexFlags{ForceZigzag: true}
   }
-|
-  FORCE_ZIGZAG '=' index_name
+| FORCE_ZIGZAG '=' index_name
   {
      $$.val = &tree.IndexFlags{ZigzagIndexes: []tree.UnrestrictedName{tree.UnrestrictedName($3)}}
   }
-|
-  FORCE_ZIGZAG '=' '[' iconst64 ']'
+| FORCE_ZIGZAG '=' '[' iconst64 ']'
   {
     /* SKIP DOC */
      $$.val = &tree.IndexFlags{ZigzagIndexIDs: []tree.IndexID{tree.IndexID($4.int64())}}
@@ -16540,6 +16539,7 @@ unreserved_keyword:
 | FORCE_NULL
 | FORCE_QUOTE
 | FORCE_INDEX
+| FORCE_INVERTED_INDEX
 | FORCE_ZIGZAG
 | FORWARD
 | FREEZE
@@ -17044,6 +17044,7 @@ bare_label_keywords:
 | FORCE_NULL
 | FORCE_QUOTE
 | FORCE_INDEX
+| FORCE_INVERTED_INDEX
 | FORCE_ZIGZAG
 | FOREIGN
 | FORMAT