[WIP] opt: add hint to ignore preserved-multiplicity consistency

docs/generated/sql/bnf/stmt_block.bnf

@@ -1127,6 +1127,7 @@ unreserved_keyword ::=
 	| 'NOWAIT'
 	| 'NULLS'
 	| 'IGNORE_FOREIGN_KEYS'
+	| 'IGNORE_PRESERVED_CONSISTENCY'
 	| 'INSENSITIVE'
 	| 'OF'
 	| 'OFF'

pkg/sql/opt/memo/multiplicity_builder.go

@@ -219,6 +219,9 @@ func filtersMatchLeftRowsAtMostOnce(left, right RelExpr, filters FiltersExpr) bo
 // according to the join filters. This is true when the following conditions are
 // satisfied:
 //
+// 0. No table on the right side is using IgnorePreservedConsistency (e.g.
+//    this is not a SELECT FOR UPDATE SKIP LOCKED).
+//
 // 1. If this is a cross join (there are no filters), then either:
 //   a. The minimum cardinality of the right input is greater than zero. There
 //      must be at least one right row for the left rows to be preserved.
@@ -257,6 +260,9 @@ func filtersMatchLeftRowsAtMostOnce(left, right RelExpr, filters FiltersExpr) bo
 // columns in the foreign key must be not-null in order to guarantee that all
 // rows will have a match in the referenced table.
 func filtersMatchAllLeftRows(left, right RelExpr, filters FiltersExpr) bool {
+	if CheckIgnorePreservedConsistency(right.Memo().Metadata(), right) {
+		return false
+	}
 	if filters.IsTrue() {
 		// Cross join case.
 		if !right.Relational().Cardinality.CanBeZero() {
@@ -446,6 +452,17 @@ func rightHasSingleFilterThatMatchesLeft(left, right RelExpr, leftCol, rightCol
 	return leftConst == rightConst
 }
 
+func CheckIgnorePreservedConsistency(md *opt.Metadata, right RelExpr) (ignore bool) {
+	right.Relational().OutputCols.ForEach(func(colID opt.ColumnID) {
+		tableID := md.ColumnMeta(colID).Table
+		if tableID != 0 && md.TableMeta(tableID).IgnorePreservedConsistency {
+			ignore = true
+			return
+		}
+	})
+	return
+}
+
 // checkSelfJoinCase returns true if all equalities in the given FiltersExpr
 // are between columns from the same position in the same base table. Panics
 // if verifyFilters is not checked first.

pkg/sql/opt/metadata.go

@@ -486,15 +486,16 @@ func (md *Metadata) DuplicateTable(
 	}
 
 	md.tables = append(md.tables, TableMeta{
-		MetaID:                   newTabID,
-		Table:                    tabMeta.Table,
-		Alias:                    tabMeta.Alias,
-		IgnoreForeignKeys:        tabMeta.IgnoreForeignKeys,
-		Constraints:              constraints,
-		ComputedCols:             computedCols,
-		partialIndexPredicates:   partialIndexPredicates,
-		indexPartitionLocalities: tabMeta.indexPartitionLocalities,
-		checkConstraintsStats:    checkConstraintsStats,
+		MetaID:                     newTabID,
+		Table:                      tabMeta.Table,
+		Alias:                      tabMeta.Alias,
+		IgnoreForeignKeys:          tabMeta.IgnoreForeignKeys,
+		IgnorePreservedConsistency: tabMeta.IgnorePreservedConsistency,
+		Constraints:                constraints,
+		ComputedCols:               computedCols,
+		partialIndexPredicates:     partialIndexPredicates,
+		indexPartitionLocalities:   tabMeta.indexPartitionLocalities,
+		checkConstraintsStats:      checkConstraintsStats,
 	})
 
 	return newTabID

pkg/sql/opt/norm/general_funcs.go

@@ -933,6 +933,10 @@ func (c *CustomFuncs) JoinPreservesRightRows(join memo.RelExpr) bool {
 	return mult.JoinPreservesRightRows(join.Op())
 }
 
+func (c *CustomFuncs) IndexJoinPreservesRows(expr memo.RelExpr) bool {
+	return !memo.CheckIgnorePreservedConsistency(c.mem.Metadata(), expr)
+}
+
 // NoJoinHints returns true if no hints were specified for this join.
 func (c *CustomFuncs) NoJoinHints(p *memo.JoinPrivate) bool {
 	return p.Flags.Empty()

pkg/sql/opt/norm/testdata/rules/limit

@@ -1571,3 +1571,11 @@ distinct-on
       │    └── column1
       └── first-agg
            └── "?column?"
+
+exec-ddl
+CREATE TABLE m (n INT PRIMARY KEY, o INT, p INT, INDEX (p))
+----
+
+optsteps
+SELECT * FROM m WHERE p > 5 AND p < 100 ORDER BY p LIMIT 5
+----

pkg/sql/opt/optbuilder/select.go

@@ -461,6 +461,9 @@ func (b *Builder) buildScan(
 		if indexFlags.IgnoreUniqueWithoutIndexKeys {
 			tabMeta.IgnoreUniqueWithoutIndexKeys = true
 		}
+		if indexFlags.IgnorePreservedConsistency {
+			tabMeta.IgnorePreservedConsistency = true
+		}
 	}
 
 	outScope = inScope.push()

pkg/sql/opt/table_meta.go

@@ -141,6 +141,12 @@ type TableMeta struct {
 	// depend on the consistency of unique without index constraints.
 	IgnoreUniqueWithoutIndexKeys bool
 
+	// IgnorePreservedConsistency is true if we should disable any rules that
+	// depend on preserved-multiplicity consistency of this table (i.e. rules that
+	// assume there is always a PK row for every secondary index row or a FK row
+	// for every FK reference). This is used for SKIP LOCKED queries.
+	IgnorePreservedConsistency bool
+
 	// Constraints stores a *FiltersExpr containing filters that are known to
 	// evaluate to true on the table data. This list is extracted from validated
 	// check constraints; specifically, those check constraints that we can prove
@@ -202,6 +208,7 @@ func (tm *TableMeta) copyFrom(from *TableMeta, copyScalarFn func(Expr) Expr) {
 		Alias:                        from.Alias,
 		IgnoreForeignKeys:            from.IgnoreForeignKeys,
 		IgnoreUniqueWithoutIndexKeys: from.IgnoreUniqueWithoutIndexKeys,
+		IgnorePreservedConsistency:   from.IgnorePreservedConsistency,
 		// Annotations are not copied.
 	}
 

pkg/sql/opt/xform/groupby_funcs.go

@@ -389,9 +389,11 @@ func (c *CustomFuncs) GenerateLimitedGroupByScans(
 	}
 	// Iterate over all non-inverted and non-partial secondary indexes.
 	var pkCols opt.ColSet
-	var iter scanIndexIter
 	var sb indexScanBuilder
 	sb.Init(c, sp.Table)
+	tabMeta := c.e.mem.Metadata().TableMeta(sp.Table)
+
+	var iter scanIndexIter
 	iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, sp, nil /* filters */, rejectPrimaryIndex|rejectInvertedIndexes)
 	iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool, constProj memo.ProjectionsExpr) {
 		// The iterator only produces pseudo-partial indexes (the predicate is
@@ -411,6 +413,12 @@ func (c *CustomFuncs) GenerateLimitedGroupByScans(
 			return
 		}
 
+		// Otherwise, try to construct an IndexJoin operator that provides the
+		// columns missing from the index.
+		if sp.Flags.NoIndexJoin || tabMeta.IgnorePreservedConsistency {
+			return
+		}
+
 		// Calculate the PK columns once.
 		if pkCols.Empty() {
 			pkCols = c.PrimaryKeyCols(sp.Table)

pkg/sql/opt/xform/limit_funcs.go

@@ -107,6 +107,7 @@ func (c *CustomFuncs) GenerateLimitedScans(
 	var pkCols opt.ColSet
 	var sb indexScanBuilder
 	sb.Init(c, scanPrivate.Table)
+	tabMeta := c.e.mem.Metadata().TableMeta(scanPrivate.Table)
 
 	// Iterate over all non-inverted, non-partial indexes, looking for those
 	// that can be limited.
@@ -145,7 +146,7 @@ func (c *CustomFuncs) GenerateLimitedScans(
 
 		// Otherwise, try to construct an IndexJoin operator that provides the
 		// columns missing from the index.
-		if scanPrivate.Flags.NoIndexJoin {
+		if scanPrivate.Flags.NoIndexJoin || tabMeta.IgnorePreservedConsistency {
 			return
 		}
 
@@ -276,9 +277,11 @@ func (c *CustomFuncs) GenerateLimitedTopKScans(
 	}
 	// Iterate over all non-inverted and non-partial secondary indexes.
 	var pkCols opt.ColSet
-	var iter scanIndexIter
 	var sb indexScanBuilder
 	sb.Init(c, sp.Table)
+	tabMeta := c.e.mem.Metadata().TableMeta(sp.Table)
+
+	var iter scanIndexIter
 	iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, sp, nil /* filters */, rejectPrimaryIndex|rejectInvertedIndexes)
 	iter.ForEach(func(index cat.Index, filters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool, constProj memo.ProjectionsExpr) {
 		// The iterator only produces pseudo-partial indexes (the predicate is
@@ -298,6 +301,12 @@ func (c *CustomFuncs) GenerateLimitedTopKScans(
 			return
 		}
 
+		// Otherwise, try to construct an IndexJoin operator that provides the
+		// columns missing from the index.
+		if sp.Flags.NoIndexJoin || tabMeta.IgnorePreservedConsistency {
+			return
+		}
+
 		// Calculate the PK columns once.
 		if pkCols.Empty() {
 			pkCols = c.PrimaryKeyCols(sp.Table)

pkg/sql/opt/xform/rules/limit.opt

@@ -41,7 +41,8 @@
 # TODO(radu): we can similarly push Offset too.
 [PushLimitIntoIndexJoin, Explore]
 (Limit
-    (IndexJoin $input:* $indexJoinPrivate:*)
+    (IndexJoin $input:* $indexJoinPrivate:*) &
+        (IndexJoinPreservesRows $input)
     $limitExpr:(Const $limit:* & (IsPositiveInt $limit))
     $ordering:* &
         (OrderingCanProjectCols

pkg/sql/opt/xform/select_funcs.go

@@ -102,11 +102,13 @@ func (c *CustomFuncs) GeneratePartialIndexScans(
 ) {
 	// Iterate over all partial indexes.
 	var pkCols opt.ColSet
+	var sb indexScanBuilder
+	sb.Init(c, scanPrivate.Table)
+	tabMeta := c.e.mem.Metadata().TableMeta(scanPrivate.Table)
+
 	var iter scanIndexIter
 	iter.Init(c.e.evalCtx, c.e.f, c.e.mem, &c.im, scanPrivate, filters, rejectNonPartialIndexes|rejectInvertedIndexes)
 	iter.ForEach(func(index cat.Index, remainingFilters memo.FiltersExpr, indexCols opt.ColSet, isCovering bool, constProj memo.ProjectionsExpr) {
-		var sb indexScanBuilder
-		sb.Init(c, scanPrivate.Table)
 		newScanPrivate := *scanPrivate
 		newScanPrivate.Index = index.Ordinal()
 		newScanPrivate.Cols = indexCols.Intersection(scanPrivate.Cols)
@@ -121,6 +123,12 @@ func (c *CustomFuncs) GeneratePartialIndexScans(
 			return
 		}
 
+		// Otherwise, try to construct an IndexJoin operator that provides the
+		// columns missing from the index.
+		if scanPrivate.Flags.NoIndexJoin || tabMeta.IgnorePreservedConsistency {
+			return
+		}
+
 		// Calculate the PK columns once.
 		if pkCols.Empty() {
 			pkCols = c.PrimaryKeyCols(scanPrivate.Table)
@@ -464,7 +472,7 @@ func (c *CustomFuncs) GenerateConstrainedScans(
 
 		// Otherwise, construct an IndexJoin operator that provides the columns
 		// missing from the index.
-		if scanPrivate.Flags.NoIndexJoin {
+		if scanPrivate.Flags.NoIndexJoin || tabMeta.IgnorePreservedConsistency {
 			return
 		}
 
@@ -892,6 +900,10 @@ func (c *CustomFuncs) GenerateInvertedIndexScans(
 		newScanPrivate.SetConstraint(c.e.evalCtx, constraint)
 		newScanPrivate.InvertedConstraint = spansToRead
 
+		if scanPrivate.Flags.NoIndexJoin || tabMeta.IgnorePreservedConsistency {
+			return
+		}
+
 		// Calculate the PK columns once.
 		if pkCols.Empty() {
 			pkCols = c.PrimaryKeyCols(scanPrivate.Table)

pkg/sql/parser/sql.y

@@ -838,7 +838,8 @@ func (u *sqlSymUnion) asTenantClause() tree.TenantID {
 %token <str> HAVING HASH HEADER HIGH HISTOGRAM HOLD HOUR
 
 %token <str> IDENTITY
-%token <str> IF IFERROR IFNULL IGNORE_FOREIGN_KEYS ILIKE IMMEDIATE IMPORT IN INCLUDE
+%token <str> IF IFERROR IFNULL IGNORE_FOREIGN_KEYS IGNORE_PRESERVED_CONSISTENCY ILIKE IMMEDIATE
+%token <str> IMPORT IN INCLUDE
 %token <str> INCLUDING INCREMENT INCREMENTAL INCREMENTAL_LOCATION
 %token <str> INET INET_CONTAINED_BY_OR_EQUALS
 %token <str> INET_CONTAINS_OR_EQUALS INDEX INDEXES INHERITS INJECT INITIALLY
@@ -10664,6 +10665,12 @@ index_flags_param:
     /* SKIP DOC */
     $$.val = &tree.IndexFlags{IgnoreForeignKeys: true}
   }
+|
+  IGNORE_PRESERVED_CONSISTENCY
+  {
+    /* SKIP DOC */
+    $$.val = &tree.IndexFlags{IgnorePreservedConsistency: true}
+  }
 |
   FORCE_ZIGZAG
   {
@@ -10741,6 +10748,7 @@ opt_index_flags:
 //   '{' NO_ZIGZAG_JOIN [, ...] '}'
 //   '{' NO_FULL_SCAN [, ...] '}'
 //   '{' IGNORE_FOREIGN_KEYS [, ...] '}'
+//   '{' IGNORE_PRESERVED_CONSISTENCY [, ...] '}'
 //   '{' FORCE_ZIGZAG = <idxname> [, ...]  '}'
 //
 // Join types:
@@ -14255,6 +14263,7 @@ unreserved_keyword:
 | NOWAIT
 | NULLS
 | IGNORE_FOREIGN_KEYS
+| IGNORE_PRESERVED_CONSISTENCY
 | INSENSITIVE
 | OF
 | OFF

pkg/sql/sem/tree/select.go

@@ -282,6 +282,7 @@ type IndexID = catid.IndexID
 //  - NO_ZIGZAG_JOIN
 //  - NO_FULL_SCAN
 //  - IGNORE_FOREIGN_KEYS
+//  - IGNORE_PRESERVED_CONSISTENCY
 //  - FORCE_ZIGZAG
 //  - FORCE_ZIGZAG=<index_name|index_id>*
 // It is used optionally after a table name in SELECT statements.
@@ -304,6 +305,11 @@ type IndexFlags struct {
 	// IgnoreUniqueWithoutIndexKeys disables optimizations based on unique without
 	// index constraints.
 	IgnoreUniqueWithoutIndexKeys bool
+	// IgnorePreservedConsistency disables optimizations based on
+	// preserved-multiplicity consistency of this table (e.g. consistency of
+	// primary and secondary index rows, consistency of foreign key
+	// references). This is used for SKIP LOCKED queries.
+	IgnorePreservedConsistency bool
 	// Zigzag hinting fields are distinct:
 	// ForceZigzag means we saw a TABLE@{FORCE_ZIGZAG}
 	// ZigzagIndexes means we saw TABLE@{FORCE_ZIGZAG=name}
@@ -345,6 +351,8 @@ func (ih *IndexFlags) CombineWith(other *IndexFlags) error {
 	result.IgnoreForeignKeys = ih.IgnoreForeignKeys || other.IgnoreForeignKeys
 	result.IgnoreUniqueWithoutIndexKeys = ih.IgnoreUniqueWithoutIndexKeys ||
 		other.IgnoreUniqueWithoutIndexKeys
+	result.IgnorePreservedConsistency = ih.IgnorePreservedConsistency ||
+		other.IgnorePreservedConsistency
 
 	if other.Direction != 0 {
 		if ih.Direction != 0 {
@@ -472,6 +480,11 @@ func (ih *IndexFlags) Format(ctx *FmtCtx) {
 			ctx.WriteString("IGNORE_UNIQUE_WITHOUT_INDEX_KEYS")
 		}
 
+		if ih.IgnorePreservedConsistency {
+			sep()
+			ctx.WriteString("IGNORE_PRESERVED_CONSISTENCY")
+		}
+
 		if ih.ForceZigzag || len(ih.ZigzagIndexes) > 0 || len(ih.ZigzagIndexIDs) > 0 {
 			sep()
 			if ih.ForceZigzag {