Merge pull request #93770 from cockroachdb/blathers/backport-release-…

…22.2-93673

release-22.2: opt: allow lookup joins to preserve index ordering with DESC columns

pkg/kv/kvclient/kvstreamer/streamer.go

@@ -429,11 +429,7 @@ func (s *Streamer) Init(
 //
 // In InOrder operation mode, responses will be delivered in reqs order. When
 // more than one row is returned for a given request, the rows for that request
-// will be sorted in the order of the lookup index if the index contains only
-// ascending columns.
-// TODO(drewk): lift the restriction that index columns must be ASC in order to
-//
-//	return results in lookup order.
+// will be sorted in the order of the lookup index.
 //
 // It is the caller's responsibility to ensure that the memory footprint of reqs
 // (i.e. roachpb.Spans inside of the requests) is reasonable. Enqueue will

pkg/sql/distsql_physical_planner.go

@@ -2522,17 +2522,14 @@ func (dsp *DistSQLPlanner) createPlanForLookupJoin(
 
 	// If any of the ordering columns originate from the lookup table, this is a
 	// case where we are ordering on a prefix of input columns followed by the
-	// lookup columns. We need to maintain the index ordering on each lookup.
+	// lookup columns.
 	var maintainLookupOrdering bool
 	numInputCols := len(plan.GetResultTypes())
 	for i := range n.reqOrdering {
 		if n.reqOrdering[i].ColIdx >= numInputCols {
+			// We need to maintain the index ordering on each lookup.
 			maintainLookupOrdering = true
-			if n.reqOrdering[i].Direction == encoding.Descending {
-				// Validate that an ordering on lookup columns does not contain
-				// descending columns.
-				panic(errors.AssertionFailedf("ordering on a lookup index with descending columns"))
-			}
+			break
 		}
 	}
 

pkg/sql/execinfrapb/processors_sql.proto

@@ -395,9 +395,7 @@ message JoinReaderSpec {
   // only be set to true if maintain_ordering is also true.
   // maintain_lookup_ordering can be used if the output needs to be ordered by
   // a prefix of input columns followed by index (lookup) columns without
-  // requiring a (buffered) sort. As an additional restriction due to
-  // implementation details, maintain_lookup_ordering can only be used when the
-  // index columns that participate in the output ordering are all ASC.
+  // requiring a (buffered) sort.
   optional bool maintain_lookup_ordering = 22 [(gogoproto.nullable) = false];
 }
 

pkg/sql/logictest/testdata/logic_test/lookup_join

@@ -843,10 +843,10 @@ WHERE views.chat_id = 1 and views.user_id = 1;
 # have to sort its output).
 
 statement ok
-CREATE TABLE xyz (x INT, y INT, z INT, PRIMARY KEY(x, y, z));
+CREATE TABLE xyz (x INT, y INT, z INT, PRIMARY KEY(x, y DESC, z));
 
 statement ok
-CREATE TABLE uvw (u INT, v INT, w INT, PRIMARY KEY(u, v, w));
+CREATE TABLE uvw (u INT, v INT, w INT, PRIMARY KEY(u, v, w DESC));
 
 statement ok
 INSERT INTO xyz VALUES (1, 1, 1), (1, 1, 2), (1, 2, 3), (2, 1, 4), (2, 1, 5), (2, 1, 6), (3, 1, 7);
@@ -855,86 +855,86 @@ statement ok
 INSERT INTO uvw VALUES (1, 1, 1), (1, 2, 2), (1, 2, 3), (2, 1, 4), (2, 1, 5), (2, 2, 6), (2, 2, 7);
 
 query IIIIII colnames
-SELECT * FROM xyz INNER LOOKUP JOIN uvw ON x = u ORDER BY x, y, z, u, v, w
+SELECT * FROM xyz INNER LOOKUP JOIN uvw ON x = u ORDER BY x, y DESC, z, u, v, w DESC
 ----
 x  y  z  u  v  w
+1  2  3  1  1  1
+1  2  3  1  2  3
+1  2  3  1  2  2
 1  1  1  1  1  1
-1  1  1  1  2  2
 1  1  1  1  2  3
+1  1  1  1  2  2
 1  1  2  1  1  1
-1  1  2  1  2  2
 1  1  2  1  2  3
-1  2  3  1  1  1
-1  2  3  1  2  2
-1  2  3  1  2  3
-2  1  4  2  1  4
+1  1  2  1  2  2
 2  1  4  2  1  5
-2  1  4  2  2  6
+2  1  4  2  1  4
 2  1  4  2  2  7
-2  1  5  2  1  4
+2  1  4  2  2  6
 2  1  5  2  1  5
-2  1  5  2  2  6
+2  1  5  2  1  4
 2  1  5  2  2  7
-2  1  6  2  1  4
+2  1  5  2  2  6
 2  1  6  2  1  5
-2  1  6  2  2  6
+2  1  6  2  1  4
 2  1  6  2  2  7
+2  1  6  2  2  6
 
 query IIIIII colnames
-SELECT * FROM xyz INNER HASH JOIN uvw ON x = u ORDER BY x, y, z, u, v, w
+SELECT * FROM xyz INNER HASH JOIN uvw ON x = u ORDER BY x, y DESC, z, u, v, w DESC
 ----
 x  y  z  u  v  w
+1  2  3  1  1  1
+1  2  3  1  2  3
+1  2  3  1  2  2
 1  1  1  1  1  1
-1  1  1  1  2  2
 1  1  1  1  2  3
+1  1  1  1  2  2
 1  1  2  1  1  1
-1  1  2  1  2  2
 1  1  2  1  2  3
-1  2  3  1  1  1
-1  2  3  1  2  2
-1  2  3  1  2  3
-2  1  4  2  1  4
+1  1  2  1  2  2
 2  1  4  2  1  5
-2  1  4  2  2  6
+2  1  4  2  1  4
 2  1  4  2  2  7
-2  1  5  2  1  4
+2  1  4  2  2  6
 2  1  5  2  1  5
-2  1  5  2  2  6
+2  1  5  2  1  4
 2  1  5  2  2  7
-2  1  6  2  1  4
+2  1  5  2  2  6
 2  1  6  2  1  5
-2  1  6  2  2  6
+2  1  6  2  1  4
 2  1  6  2  2  7
+2  1  6  2  2  6
 
 query IIIIII colnames
-SELECT * FROM xyz INNER LOOKUP JOIN uvw ON x = u AND y = v ORDER BY u, x, v, y, z, w
+SELECT * FROM xyz INNER LOOKUP JOIN uvw ON x = u AND y = v ORDER BY u, x, v, y DESC, z, w DESC
 ----
 x  y  z  u  v  w
 1  1  1  1  1  1
 1  1  2  1  1  1
-1  2  3  1  2  2
 1  2  3  1  2  3
-2  1  4  2  1  4
+1  2  3  1  2  2
 2  1  4  2  1  5
-2  1  5  2  1  4
+2  1  4  2  1  4
 2  1  5  2  1  5
-2  1  6  2  1  4
+2  1  5  2  1  4
 2  1  6  2  1  5
+2  1  6  2  1  4
 
 query IIIIII colnames
-SELECT * FROM xyz INNER HASH JOIN uvw ON x = u AND y = v ORDER BY u, x, v, y, z, w
+SELECT * FROM xyz INNER HASH JOIN uvw ON x = u AND y = v ORDER BY u, x, v, y DESC, z, w DESC
 ----
 x  y  z  u  v  w
 1  1  1  1  1  1
 1  1  2  1  1  1
-1  2  3  1  2  2
 1  2  3  1  2  3
-2  1  4  2  1  4
+1  2  3  1  2  2
 2  1  4  2  1  5
-2  1  5  2  1  4
+2  1  4  2  1  4
 2  1  5  2  1  5
-2  1  6  2  1  4
+2  1  5  2  1  4
 2  1  6  2  1  5
+2  1  6  2  1  4
 
 # Test inequality lookup joins.
 # Case with idxCol <= inputCol.

pkg/sql/opt/ordering/lookup_join.go

@@ -204,24 +204,22 @@ func lookupJoinBuildProvided(expr memo.RelExpr, required *props.OrderingChoice)
 //
 // It is possible for a lookup join to supply an ordering that references index
 // columns if the ordering consists of a series of input columns that form a key
-// over the input, followed by the index columns in index order. Due to
-// implementation details, currently the ordering columns from the index must be
-// ASC. The following is a case where a lookup join could maintain an ordering
-// over both input and index columns:
+// over the input, followed by the index columns in index order. The following
+// is a case where a lookup join could maintain an ordering over both input and
+// index columns:
 //
 //	CREATE TABLE ab (a INT, b INT, PRIMARY KEY(a, b));
-//	CREATE TABLE xyz (x INT, y INT, z INT, PRIMARY KEY(x, y, z DESC));
-//	SELECT * FROM ab INNER LOOKUP JOIN xy ON a = x ORDER BY a, b, x, y;
+//	CREATE TABLE xy (x INT, y INT, PRIMARY KEY(x, y DESC));
+//	SELECT * FROM ab INNER LOOKUP JOIN xy ON a = x ORDER BY a, b, x, y DESC;
 //
 // Note that in this example the 'a' and 'b' columns form a key over the
 // input of the lookup join. Additionally, the 'x' column alone is not a key
 // for the 'xy' table, so each lookup may return multiple rows (which need
-// to be ordered among themselves). Since the postfix of the ordering that
-// references index columns is in index order (x, y) and has no DESC
-// columns, the lookup join in the example can supply the ordering itself.
-// On the other hand, switching 'b' and 'y' in the ordering, removing 'b',
-// or adding the 'z' column to the required order would mean the query would
-// require a sort.
+// to be ordered among themselves). Since the suffix of the ordering that
+// references index columns is in index order (x, y DESC), the lookup join in
+// the example can supply the ordering itself. On the other hand, switching
+// 'b' and 'y' in the ordering, removing 'b', or changing the ordering on 'y' to
+// ASC would mean the query would require a sort.
 //
 // Note that the Columns field of the required OrderingChoice should reflect the
 // postfix of the required ordering that cannot be satisfied by input columns,
@@ -236,11 +234,13 @@ func getLookupOrdCols(
 		// joins can only maintain the index ordering for each individual input
 		// row, so we need to disallow cases where different input rows may sort
 		// the same on the input ordering.
-		// TODO(drewk): it is possible to take advantage of the index ordering
-		//  when the input ordering does not form a key over the input. In this
-		//  case, we would require that the index ordering columns for a given
-		//  input row are functionally determined by the input ordering columns.
-		//  This would disqualify IN constraints and inequalities.
+		//
+		// Note that it would be technically correct to use the index ordering when
+		// the input ordering does not form a key over the input iff the input
+		// ordering columns functionally determined the index ordering columns.
+		// However, in this case the addition of the index ordering columns would be
+		// trivial, since the ordering could be simplified to just include the input
+		// ordering columns (see OrderingChoice.Simplify).
 		return nil, false
 	}
 	// The columns from the prefix of the required ordering satisfied by the
@@ -272,11 +272,6 @@ func getLookupOrdCols(
 			// satisfy the required ordering, so break instead of returning.
 			break
 		}
-		if idx.Column(i).Descending {
-			// The index ordering columns must be ASC in order for lookups to be
-			// returned in index order.
-			return nil, false
-		}
 		indexOrder = append(indexOrder, opt.MakeOrderingColumn(idxColID, idx.Column(i).Descending))
 	}
 	// Check if the index ordering satisfies the postfix of the required

pkg/sql/opt/ordering/lookup_join_test.go

@@ -137,15 +137,15 @@ func TestLookupJoinProvided(t *testing.T) {
 			input:    "+5",
 			provided: "+1,+2",
 		},
-		{ // case 8: the lookup join preserves the input ordering but cannot provide
-			// the entire required ordering because the index has a descending column.
+		{ // case 8: the lookup join preserves the input ordering and maintains the
+			// ordering of the descending index on lookups. Joining on c1 = c5.
 			index:    descendingIndex,
 			keyCols:  opt.ColList{5},
 			inputKey: c(5, 6),
 			outCols:  c(2, 3, 4, 5, 6),
 			required: "+(1|5),+6,-2",
 			input:    "+5,+6",
-			provided: "+5,+6",
+			provided: "+5,+6,-2",
 		},
 	}
 
@@ -317,14 +317,13 @@ func TestLookupJoinCanProvide(t *testing.T) {
 			required:   "+(1|5),+6,-2",
 			canProvide: false,
 		},
-		{ // Case 11: the ordering cannot be satisfied because the lookup index has
-			// a descending column.
+		{ // Case 11: an ordering with a descending column can be satisfied..
 			idx:        descendingIndex,
 			keyCols:    opt.ColList{5},
 			outCols:    c(1, 2, 5, 6),
 			inputKey:   c(5, 6),
 			required:   "+(1|5),+6,-2",
-			canProvide: false,
+			canProvide: true,
 		},
 		{ // Case 12: the ordering cannot be satisfied because the required ordering
 			// is missing index column c1.

pkg/sql/opt/xform/testdata/physprops/ordering

@@ -2715,8 +2715,7 @@ inner-join (lookup abc)
  │    └── ordering: +1,+2,+3
  └── filters (true)
 
-# Can supply the requested ordering because the descending column from the
-# index does not take part in the ordering (no sort should be added).
+# Preserving lookup ordering (no sort should be added).
 opt
 SELECT * FROM xyz INNER LOOKUP JOIN abc@abc_desc ON x = a ORDER BY x, y, z, a, b
 ----
@@ -2733,6 +2732,24 @@ inner-join (lookup abc@abc_desc)
  │    └── ordering: +1,+2,+3
  └── filters (true)
 
+# Preserving lookup ordering (no sort should be added). Index order includes a
+# descending column.
+opt
+SELECT * FROM xyz INNER LOOKUP JOIN abc@abc_desc ON x = a ORDER BY x, y, z, a, b, c DESC
+----
+inner-join (lookup abc@abc_desc)
+ ├── columns: x:1!null y:2!null z:3!null a:6!null b:7!null c:8!null
+ ├── flags: force lookup join (into right side)
+ ├── key columns: [1] = [6]
+ ├── key: (2,3,6-8)
+ ├── fd: (1)==(6), (6)==(1)
+ ├── ordering: +(1|6),+2,+3,+7,-8 [actual: +1,+2,+3,+7,-8]
+ ├── scan xyz
+ │    ├── columns: x:1!null y:2!null z:3!null
+ │    ├── key: (1-3)
+ │    └── ordering: +1,+2,+3
+ └── filters (true)
+
 # Cannot supply requested ordering because input and lookup ordering columns
 # are interleaved.
 opt
@@ -2803,7 +2820,7 @@ sort (segmented)
 # Cannot supply the requested ordering because the direction of the 'c' column
 # is not the same as in the index.
 opt
-SELECT * FROM xyz INNER LOOKUP JOIN abc ON x = a ORDER BY x, y, z, b, c DESC
+SELECT * FROM xyz INNER LOOKUP JOIN abc@primary ON x = a ORDER BY x, y, z, b, c DESC
 ----
 sort (segmented)
  ├── columns: x:1!null y:2!null z:3!null a:6!null b:7!null c:8!null
@@ -2823,29 +2840,6 @@ sort (segmented)
       │    └── ordering: +1,+2,+3
       └── filters (true)
 
-# Cannot supply the requested ordering because the descending column from the
-# index shows up in the ordering.
-opt
-SELECT * FROM xyz INNER LOOKUP JOIN abc@abc_desc ON x = a ORDER BY x, y, z, a, b, c DESC
-----
-sort (segmented)
- ├── columns: x:1!null y:2!null z:3!null a:6!null b:7!null c:8!null
- ├── key: (2,3,6-8)
- ├── fd: (1)==(6), (6)==(1)
- ├── ordering: +(1|6),+2,+3,+7,-8 [actual: +1,+2,+3,+7,-8]
- └── inner-join (lookup abc@abc_desc)
-      ├── columns: x:1!null y:2!null z:3!null a:6!null b:7!null c:8!null
-      ├── flags: force lookup join (into right side)
-      ├── key columns: [1] = [6]
-      ├── key: (2,3,6-8)
-      ├── fd: (1)==(6), (6)==(1)
-      ├── ordering: +1,+2,+3
-      ├── scan xyz
-      │    ├── columns: x:1!null y:2!null z:3!null
-      │    ├── key: (1-3)
-      │    └── ordering: +1,+2,+3
-      └── filters (true)
-
 # Regression test for #85393 - use only columns from the required ordering when
 # building the provided ordering for Project operators.
 exec-ddl