sql: add DisableNotVisibleIndex to ScanFlags

This commit adds a flag to ScanFlags to indicate when the invisible index
feature should be enabled in buildScan. The invisible index feature should be
enabled by default but temporarily disabled during any unique or foreign key
constraint check. More details on how the decision was made and which flag to
pass in `docs/RFCS/20220628_invisible_index.md`. Note that no logic has been
added to the optimizer yet. This commit just passes the correct flag to
buildScan and shows the effect of the flag in the output of test data.

Assists: #72576

See also: #85354

Release note: none

pkg/sql/explain_bundle.go

@@ -244,9 +244,9 @@ func (b *stmtBundleBuilder) addOptPlans() {
 
 	b.z.AddFile("opt.txt", formatOptPlan(memo.ExprFmtHideAll))
 	b.z.AddFile("opt-v.txt", formatOptPlan(
-		memo.ExprFmtHideQualifications|memo.ExprFmtHideScalars|memo.ExprFmtHideTypes,
+		memo.ExprFmtHideQualifications|memo.ExprFmtHideScalars|memo.ExprFmtHideTypes|memo.ExprFmtHideNotVisibleIndexInfo,
 	))
-	b.z.AddFile("opt-vv.txt", formatOptPlan(memo.ExprFmtHideQualifications))
+	b.z.AddFile("opt-vv.txt", formatOptPlan(memo.ExprFmtHideQualifications|memo.ExprFmtHideNotVisibleIndexInfo))
 }
 
 // addExecPlan adds the EXPLAIN (VERBOSE) plan as file plan.txt.

pkg/sql/opt/exec/execbuilder/relational.go

@@ -1006,7 +1006,8 @@ func (b *Builder) buildApplyJoin(join memo.RelExpr) (execPlan, error) {
 			if errors.IsAssertionFailure(err) {
 				// Enhance the error with the EXPLAIN (OPT, VERBOSE) of the inner
 				// expression.
-				fmtFlags := memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars | memo.ExprFmtHideTypes
+				fmtFlags := memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars | memo.ExprFmtHideTypes |
+					memo.ExprFmtHideNotVisibleIndexInfo
 				explainOpt := o.FormatExpr(newRightSide, fmtFlags)
 				err = errors.WithDetailf(err, "newRightSide:\n%s", explainOpt)
 			}

pkg/sql/opt/exec/execbuilder/statement.go

@@ -91,10 +91,10 @@ func (b *Builder) buildExplainOpt(explain *memo.ExplainExpr) (execPlan, error) {
 	switch {
 	case explain.Options.Flags[tree.ExplainFlagVerbose]:
 		fmtFlags = memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars |
-			memo.ExprFmtHideTypes | memo.ExprFmtHideNotNull
+			memo.ExprFmtHideTypes | memo.ExprFmtHideNotNull | memo.ExprFmtHideNotVisibleIndexInfo
 
 	case explain.Options.Flags[tree.ExplainFlagTypes]:
-		fmtFlags = memo.ExprFmtHideQualifications
+		fmtFlags = memo.ExprFmtHideQualifications | memo.ExprFmtHideNotVisibleIndexInfo
 	}
 
 	// Format the plan here and pass it through to the exec factory.

pkg/sql/opt/indexrec/indexrec_test.go

@@ -29,7 +29,8 @@ func TestIndexRec(t *testing.T) {
 	defer log.Scope(t).Close(t)
 
 	const fmtFlags = memo.ExprFmtHideStats | memo.ExprFmtHideRuleProps |
-		memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars | memo.ExprFmtHideTypes
+		memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars | memo.ExprFmtHideTypes |
+		memo.ExprFmtHideNotVisibleIndexInfo
 	datadriven.Walk(t, testutils.TestDataPath(t), func(t *testing.T, path string) {
 		catalog := testcat.New()
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {

pkg/sql/opt/memo/expr.go

@@ -368,6 +368,17 @@ type ScanFlags struct {
 	Direction   tree.Direction
 	Index       int
 
+	// When the optimizer is performing unique constraint or foreign key
+	// constraint check, we will temporarily disable the not visible index feature
+	// and treat all indexes as they are visible. Default behaviour is to enable
+	// the not visible feature. We will pass this flag, DisableNotVisibleIndex, to
+	// the memo group when we are building a memo group for a ScanOp expression on
+	// the given table. Later on, optimizer will enumerate all indexes on the
+	// table to generate equivalent memo groups. If DisableNotVisibleIndex is
+	// true, optimizer will also generate equivalent memo group using the
+	// invisible index. Otherwise, optimizer will ignore the invisible indexes.
+	DisableNotVisibleIndex bool
+
 	// ZigzagIndexes makes planner prefer a zigzag with particular indexes.
 	// ForceZigzag must also be true.
 	ZigzagIndexes util.FastIntSet
@@ -378,6 +389,12 @@ func (sf *ScanFlags) Empty() bool {
 	return *sf == ScanFlags{}
 }
 
+// onlyNotVisibleIndexFlagOn returns true if DisableNotVisibleIndex is the only
+// flag set on.
+func (sf *ScanFlags) onlyNotVisibleIndexFlagOn() bool {
+	return *sf == ScanFlags{DisableNotVisibleIndex: true}
+}
+
 // JoinFlags stores restrictions on the join execution method, derived from
 // hints for a join specified in the query (see tree.JoinTableExpr).  It is a
 // bitfield where each bit indicates if a certain type of join is disallowed or
@@ -678,6 +695,35 @@ func (s *ScanPrivate) IsVirtualTable(md *opt.Metadata) bool {
 	return tab.IsVirtualTable()
 }
 
+// IsNotVisibleIndexScan returns true if the index being scanned is a not
+// visible index.
+func (s *ScanPrivate) IsNotVisibleIndexScan(md *opt.Metadata) bool {
+	index := md.Table(s.Table).Index(s.Index)
+	return index.IsNotVisible()
+}
+
+// showNotVisibleIndexInfo is a helper function that checks if we want to show
+// invisible index info. We don't want to show the information is
+// hideShowNotVisibleIndexInfo is true, and we are not scanning an invisible
+// index. Otherwise, we want to show invisible index info.
+func (s *ScanPrivate) showNotVisibleIndexInfo(md *opt.Metadata, hideNotVisibleIndexInfo bool) bool {
+	return !(hideNotVisibleIndexInfo && !s.IsNotVisibleIndexScan(md))
+}
+
+// shouldPrintFlags is a helper function to check if we want to print the
+// "flags:" for ScanFlags formatting.
+func (s *ScanPrivate) shouldPrintFlags(md *opt.Metadata, hideNotVisibleIndexInfo bool) bool {
+	if s.Flags.Empty() {
+		return false
+	}
+	// If DisableNotVisibleIndex is the only flag on, we only want to print
+	// "flags:" if we are actually going to show invisible index info.
+	if s.Flags.onlyNotVisibleIndexFlagOn() && !s.showNotVisibleIndexInfo(md, hideNotVisibleIndexInfo) {
+		return false
+	}
+	return true
+}
+
 // IsLocking returns true if the ScanPrivate is configured to use a row-level
 // locking mode. This can be the case either because the Scan is in the scope of
 // a SELECT .. FOR [KEY] UPDATE/SHARE clause or because the Scan was configured

pkg/sql/opt/memo/expr_format.go

@@ -38,6 +38,41 @@ var ScalarFmtInterceptor func(f *ExprFmtCtx, expr opt.ScalarExpr) string
 // formatted output.
 type ExprFmtFlags int
 
+// ExprFmtFlags takes advantages of bit masking and iota. If you want to add a
+// flag here, you should only add flags to hide things but not to show things.
+// Also, you have to add the flag after ExprFmtHideMiscProps and before
+// ExprFmtHideAll.
+
+// iota is initialized as 0 at the start of const and increments by 1 every time
+// a new const is declared. In addition, expressions are also implicitly repeated.
+// For example,
+// const (                              // const (
+//	First  int = iota // 0              // 	First  int = iota * 3 // 0 * 3
+//	Second            // 1              // 	Second                // 1 * 3
+//	Third             // 2              // )
+//	Fourth            // 3
+// )
+
+// In our example here, 1 means the flag is on and 0 means the flag is off.
+//const (
+//	ExprFmtShowAll int = 0 // iota is 0, but it's not used    0000 0000
+//	ExprFmtHideMiscProps int = 1 << (iota - 1)
+//                             // iota is 1, 1 << (1 - 1) 0000 0001 = 1
+//	ExprFmtHideConstraints     // iota is 2, 1 << (2 - 1) 0000 0010 = 2
+//	ExprFmtHideFuncDeps        // iota is 3, 1 << (3 - 1) 0000 0100 = 4
+//  ...
+//  ExprFmtHideAll             // (1 << iota) - 1
+//)
+// If we want to set ExprFmtHideMiscProps and ExprFmtHideConstraints on, we
+// would have f := ExprFmtHideMiscProps | ExprFmtHideConstraints 0000 0011.
+// ExprFmtShowAll has all 0000 0000. This is because all flags represent when
+// something is hiding. If every bit is 0, that just means everything is
+// showing. ExprFmtHideAll is (1 << iota) - 1. For example, if the last const
+// iota is 4, ExprFmtHideAll would have (1 << 4) - 1 which is 0001 0000 - 1 =
+// 0000 1111 which is just setting all flags on => hiding everything. If we have
+// a flag that show things, ShowAll = 0000..000 would actually turn this flag
+// off. Thus, in order for ExprFmtShowAll and ExprFmtHideAll to be correct, we
+// can only add flags to hide things but not flags to show things.
 const (
 	// ExprFmtShowAll shows all properties of the expression.
 	ExprFmtShowAll ExprFmtFlags = 0
@@ -88,6 +123,12 @@ const (
 	// ExprFmtHideColumns removes column information.
 	ExprFmtHideColumns
 
+	// ExprFmtHideNotVisibleIndexInfo hides information about invisible index. We
+	// will only show invisible index info when we are actually scanning an
+	// invisible index. If this flag is off, we will always show invisible index
+	// info regardless of whether we are actually scanning an invisible index.
+	ExprFmtHideNotVisibleIndexInfo
+
 	// ExprFmtHideAll shows only the basic structure of the expression.
 	// Note: this flag should be used judiciously, as its meaning changes whenever
 	// we add more flags.
@@ -419,7 +460,8 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) {
 		if private.HardLimit.IsSet() {
 			tp.Childf("limit: %s", private.HardLimit)
 		}
-		if !private.Flags.Empty() {
+
+		if private.shouldPrintFlags(md, f.HasFlags(ExprFmtHideNotVisibleIndexInfo)) {
 			var b strings.Builder
 			b.WriteString("flags:")
 			if private.Flags.NoIndexJoin {
@@ -460,6 +502,10 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) {
 					}
 				}
 			}
+
+			if private.Flags.DisableNotVisibleIndex && private.showNotVisibleIndexInfo(md, f.HasFlags(ExprFmtHideNotVisibleIndexInfo)) {
+				b.WriteString(" disabled not visible index feature")
+			}
 			tp.Child(b.String())
 		}
 		f.formatLocking(tp, private.Locking)

pkg/sql/opt/memo/interner_test.go

@@ -379,7 +379,7 @@ func TestInterner(t *testing.T) {
 		{hashFn: in.hasher.HashScanFlags, eqFn: in.hasher.IsScanFlagsEqual, variations: []testVariation{
 			// Use unnamed fields so that compilation fails if a new field is
 			// added to ScanFlags.
-			{val1: ScanFlags{false, false, false, false, false, 0, 0, util.FastIntSet{}}, val2: ScanFlags{}, equal: true},
+			{val1: ScanFlags{false, false, false, false, false, 0, 0, false, util.FastIntSet{}}, val2: ScanFlags{}, equal: true},
 			{val1: ScanFlags{}, val2: ScanFlags{}, equal: true},
 			{val1: ScanFlags{NoIndexJoin: false}, val2: ScanFlags{NoIndexJoin: true}, equal: false},
 			{val1: ScanFlags{NoIndexJoin: true}, val2: ScanFlags{NoIndexJoin: true}, equal: true},

pkg/sql/opt/memo/memo_test.go

@@ -34,7 +34,7 @@ import (
 
 func TestMemo(t *testing.T) {
 	flags := memo.ExprFmtHideCost | memo.ExprFmtHideRuleProps | memo.ExprFmtHideQualifications |
-		memo.ExprFmtHideStats
+		memo.ExprFmtHideStats | memo.ExprFmtHideNotVisibleIndexInfo
 	runDataDrivenTest(t, testutils.TestDataPath(t, "memo"), flags)
 }
 
@@ -43,19 +43,20 @@ func TestFormat(t *testing.T) {
 }
 
 func TestLogicalProps(t *testing.T) {
-	flags := memo.ExprFmtHideCost | memo.ExprFmtHideQualifications | memo.ExprFmtHideStats
+	flags := memo.ExprFmtHideCost | memo.ExprFmtHideQualifications | memo.ExprFmtHideStats |
+		memo.ExprFmtHideNotVisibleIndexInfo
 	runDataDrivenTest(t, testutils.TestDataPath(t, "logprops"), flags)
 }
 
 func TestStats(t *testing.T) {
 	flags := memo.ExprFmtHideCost | memo.ExprFmtHideRuleProps | memo.ExprFmtHideQualifications |
-		memo.ExprFmtHideScalars
+		memo.ExprFmtHideScalars | memo.ExprFmtHideNotVisibleIndexInfo
 	runDataDrivenTest(t, testutils.TestDataPath(t, "stats"), flags)
 }
 
 func TestStatsQuality(t *testing.T) {
 	flags := memo.ExprFmtHideCost | memo.ExprFmtHideRuleProps | memo.ExprFmtHideQualifications |
-		memo.ExprFmtHideScalars
+		memo.ExprFmtHideScalars | memo.ExprFmtHideNotVisibleIndexInfo
 	runDataDrivenTest(t, testutils.TestDataPath(t, "stats_quality"), flags)
 }
 

pkg/sql/opt/memo/testdata/memo

@@ -128,7 +128,7 @@ WHERE a.y>1 AND a.x::string=b.x
 ORDER BY y
 LIMIT 10
 ----
-memo (optimized, ~23KB, required=[presentation: y:2,x:5,c:10] [ordering: +2])
+memo (optimized, ~24KB, required=[presentation: y:2,x:5,c:10] [ordering: +2])
  ├── G1: (project G2 G3 y x)
  │    ├── [presentation: y:2,x:5,c:10] [ordering: +2]
  │    │    ├── best: (project G2="[ordering: +2]" G3 y x)

pkg/sql/opt/memo/typing_test.go

@@ -31,7 +31,8 @@ func TestTyping(t *testing.T) {
 			memo.ExprFmtHideStats|
 			memo.ExprFmtHideCost|
 			memo.ExprFmtHideQualifications|
-			memo.ExprFmtHideScalars,
+			memo.ExprFmtHideScalars|
+			memo.ExprFmtHideNotVisibleIndexInfo,
 	)
 }
 

pkg/sql/opt/norm/norm_test.go

@@ -42,7 +42,8 @@ func TestNormRules(t *testing.T) {
 	defer log.Scope(t).Close(t)
 
 	const fmtFlags = memo.ExprFmtHideStats | memo.ExprFmtHideCost | memo.ExprFmtHideRuleProps |
-		memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars | memo.ExprFmtHideTypes
+		memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars | memo.ExprFmtHideTypes |
+		memo.ExprFmtHideNotVisibleIndexInfo
 	datadriven.Walk(t, testutils.TestDataPath(t, "rules"), func(t *testing.T, path string) {
 		catalog := testcat.New()
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
@@ -61,7 +62,8 @@ func TestNormRuleProps(t *testing.T) {
 	defer log.Scope(t).Close(t)
 
 	const fmtFlags = memo.ExprFmtHideStats | memo.ExprFmtHideCost |
-		memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars | memo.ExprFmtHideTypes
+		memo.ExprFmtHideQualifications | memo.ExprFmtHideScalars | memo.ExprFmtHideTypes |
+		memo.ExprFmtHideNotVisibleIndexInfo
 	datadriven.Walk(t, testutils.TestDataPath(t, "ruleprops"), func(t *testing.T, path string) {
 		catalog := testcat.New()
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {

pkg/sql/opt/norm/testdata/rules/with

@@ -601,6 +601,7 @@ with &2 (cte)
  │                   │         └── &1: count=1 used-columns=(6)
  │                   ├── scan t.public.parent
  │                   │    ├── columns: t.public.parent.p:8(int!null)
+ │                   │    ├── flags: disabled not visible index feature
  │                   │    ├── stats: [rows=1000, distinct(8)=1000, null(8)=0, avgsize(8)=4]
  │                   │    ├── cost: 1044.22
  │                   │    ├── key: (8)

pkg/sql/opt/optbuilder/fk_cascade.go

@@ -299,6 +299,7 @@ func (cb *onDeleteFastCascadeBuilder) Build(
 			nil, /* indexFlags */
 			noRowLocking,
 			b.allocScope(),
+			true, /* disableNotVisibleIndex */
 		)
 		mb.outScope = mb.fetchScope
 
@@ -520,6 +521,7 @@ func (b *Builder) buildDeleteCascadeMutationInput(
 		nil, /* indexFlags */
 		noRowLocking,
 		b.allocScope(),
+		true, /* disableNotVisibleIndex */
 	)
 
 	numFKCols := fk.ColumnCount()
@@ -756,6 +758,7 @@ func (b *Builder) buildUpdateCascadeMutationInput(
 		nil, /* indexFlags */
 		noRowLocking,
 		b.allocScope(),
+		true, /* disableNotVisibleIndex */
 	)
 
 	numFKCols := fk.ColumnCount()

pkg/sql/opt/optbuilder/mutation_builder.go

@@ -295,6 +295,7 @@ func (mb *mutationBuilder) buildInputForUpdate(
 		indexFlags,
 		noRowLocking,
 		inScope,
+		false, /* disableNotVisibleIndex */
 	)
 
 	// Set list of columns that will be fetched by the input expression.
@@ -410,6 +411,7 @@ func (mb *mutationBuilder) buildInputForDelete(
 		indexFlags,
 		noRowLocking,
 		inScope,
+		false, /* disableNotVisibleIndex */
 	)
 	mb.outScope = mb.fetchScope
 

pkg/sql/opt/optbuilder/mutation_builder_arbiter.go

@@ -294,6 +294,7 @@ func (mb *mutationBuilder) buildAntiJoinForDoNothingArbiter(
 		nil, /* indexFlags */
 		noRowLocking,
 		inScope,
+		true, /* disableNotVisibleIndex */
 	)
 
 	// If the index is a unique partial index, then rows that are not in the
@@ -379,6 +380,7 @@ func (mb *mutationBuilder) buildLeftJoinForUpsertArbiter(
 		nil, /* indexFlags */
 		noRowLocking,
 		inScope,
+		true, /* disableNotVisibleIndex */
 	)
 	// Set fetchColIDs to reference the columns created for the fetch values.
 	mb.setFetchColIDs(mb.fetchScope.cols)
@@ -586,6 +588,7 @@ func (h *arbiterPredicateHelper) tableScope() *scope {
 			nil, /* indexFlags */
 			noRowLocking,
 			h.mb.b.allocScope(),
+			false, /* disableNotVisibleIndex */
 		)
 	}
 	return h.tableScopeLazy

pkg/sql/opt/optbuilder/mutation_builder_fk.go

@@ -620,6 +620,7 @@ func (h *fkCheckHelper) buildOtherTableScan() (outScope *scope, tabMeta *opt.Tab
 		&tree.IndexFlags{IgnoreForeignKeys: true},
 		noRowLocking,
 		h.mb.b.allocScope(),
+		true, /* disableNotVisibleIndex */
 	), otherTabMeta
 }
 

pkg/sql/opt/optbuilder/mutation_builder_unique.go

@@ -410,6 +410,7 @@ func (h *uniqueCheckHelper) buildTableScan() (outScope *scope, ordinals []int) {
 		&tree.IndexFlags{IgnoreUniqueWithoutIndexKeys: true},
 		noRowLocking,
 		h.mb.b.allocScope(),
+		true, /* disableNotVisibleIndex */
 	), ordinals
 }
 

pkg/sql/opt/optbuilder/select.go

@@ -121,6 +121,7 @@ func (b *Builder) buildDataSource(
 					includeInverted:  false,
 				}),
 				indexFlags, locking, inScope,
+				false, /* disableNotVisibleIndex */
 			)
 
 		case cat.Sequence:
@@ -407,7 +408,8 @@ func (b *Builder) buildScanFromTableRef(
 
 	tn := tree.MakeUnqualifiedTableName(tab.Name())
 	tabMeta := b.addTable(tab, &tn)
-	return b.buildScan(tabMeta, ordinals, indexFlags, locking, inScope)
+
+	return b.buildScan(tabMeta, ordinals, indexFlags, locking, inScope, false /* disableNotVisibleIndex */)
 }
 
 // addTable adds a table to the metadata and returns the TableMeta. The table
@@ -447,6 +449,7 @@ func (b *Builder) buildScan(
 	indexFlags *tree.IndexFlags,
 	locking lockingSpec,
 	inScope *scope,
+	disableNotVisibleIndex bool,
 ) (outScope *scope) {
 	if ordinals == nil {
 		panic(errors.AssertionFailedf("no ordinals"))
@@ -594,6 +597,7 @@ func (b *Builder) buildScan(
 		private.Flags.NoIndexJoin = true
 		private.Flags.NoZigzagJoin = true
 	}
+	private.Flags.DisableNotVisibleIndex = disableNotVisibleIndex
 
 	b.addCheckConstraintsForTable(tabMeta)
 	b.addComputedColsForTable(tabMeta)