Merge #75870 #76007 #76115 #76262

75870: rfc: pgwire-compatible query cancellation r=otan,knz a=rafiss

refs #41335

Release note: None

76007: physicalplan: add support for multi-stage execution of regr_avgx, regr_avgy, regr_intercept, regr_r2, and regr_slope aggregate functions. r=yuzefovich a=mneverov

physicalplan: add support for multi-stage execution of regr_avgx, regr_avgy,
regr_intercept, regr_r2, and regr_slope aggregate functions.

See #58347.

Release note (performance improvement): regr_avgx, regr_avgy, regr_intercept,
regr_r2, and regr_slope aggregate functions are now evaluated more efficiently
in a distributed setting

76115: sql: use 16 as default bucket count for hash index r=chengxiong-ruan a=chengxiong-ruan

for some reason I forgot to modify it in my previous pr :(

Release note (sql change): 16 is used as the default bucket count
for hash sharded index.

76262: server: remove DeprecateBaseEncryptionRegistry r=jbowens a=jbowens

Remove the migration server DeprecateBaseEncryptionRegistry RPC. It was
used for the migration of the encryption-at-rest registry format in
21.2 and is no longer relevant.

Missed this in #74314.

Release note: None

Co-authored-by: Rafi Shamim <[email protected]>
Co-authored-by: Max Neverov <[email protected]>
Co-authored-by: Chengxiong Ruan <[email protected]>
Co-authored-by: Jackson Owens <[email protected]>

docs/generated/settings/settings-for-tenants.txt

@@ -85,7 +85,7 @@ sql.cross_db_views.enabled	boolean	false	if true, creating views that refer to o
 sql.defaults.cost_scans_with_default_col_size.enabled	boolean	false	setting to true uses the same size for all columns to compute scan cost
 sql.defaults.datestyle	enumeration	iso, mdy	default value for DateStyle session setting [iso, mdy = 0, iso, dmy = 1, iso, ymd = 2]
 sql.defaults.datestyle.enabled	boolean	false	default value for datestyle_enabled session setting
-sql.defaults.default_hash_sharded_index_bucket_count	integer	8	used as bucket count if bucket count is not specified in hash sharded index definition
+sql.defaults.default_hash_sharded_index_bucket_count	integer	16	used as bucket count if bucket count is not specified in hash sharded index definition
 sql.defaults.default_int_size	integer	8	the size, in bytes, of an INT type
 sql.defaults.disallow_full_table_scans.enabled	boolean	false	setting to true rejects queries that have planned a full table scan
 sql.defaults.distsql	enumeration	auto	default distributed SQL execution mode [off = 0, auto = 1, on = 2, always = 3]

docs/generated/settings/settings.html

@@ -96,7 +96,7 @@
 <tr><td><code>sql.defaults.cost_scans_with_default_col_size.enabled</code></td><td>boolean</td><td><code>false</code></td><td>setting to true uses the same size for all columns to compute scan cost</td></tr>
 <tr><td><code>sql.defaults.datestyle</code></td><td>enumeration</td><td><code>iso, mdy</code></td><td>default value for DateStyle session setting [iso, mdy = 0, iso, dmy = 1, iso, ymd = 2]</td></tr>
 <tr><td><code>sql.defaults.datestyle.enabled</code></td><td>boolean</td><td><code>false</code></td><td>default value for datestyle_enabled session setting</td></tr>
-<tr><td><code>sql.defaults.default_hash_sharded_index_bucket_count</code></td><td>integer</td><td><code>8</code></td><td>used as bucket count if bucket count is not specified in hash sharded index definition</td></tr>
+<tr><td><code>sql.defaults.default_hash_sharded_index_bucket_count</code></td><td>integer</td><td><code>16</code></td><td>used as bucket count if bucket count is not specified in hash sharded index definition</td></tr>
 <tr><td><code>sql.defaults.default_int_size</code></td><td>integer</td><td><code>8</code></td><td>the size, in bytes, of an INT type</td></tr>
 <tr><td><code>sql.defaults.disallow_full_table_scans.enabled</code></td><td>boolean</td><td><code>false</code></td><td>setting to true rejects queries that have planned a full table scan</td></tr>
 <tr><td><code>sql.defaults.distsql</code></td><td>enumeration</td><td><code>auto</code></td><td>default distributed SQL execution mode [off = 0, auto = 1, on = 2, always = 3]</td></tr>

pkg/server/migration.go

@@ -210,37 +210,6 @@ func (m *migrationServer) PurgeOutdatedReplicas(
 	return resp, nil
 }
 
-// TODO(ayang): remove this RPC and associated request/response in 22.1
-func (m *migrationServer) DeprecateBaseEncryptionRegistry(
-	ctx context.Context, req *serverpb.DeprecateBaseEncryptionRegistryRequest,
-) (*serverpb.DeprecateBaseEncryptionRegistryResponse, error) {
-	const opName = "deprecate-base-encryption-registry"
-	ctx, span := m.server.AnnotateCtxWithSpan(ctx, opName)
-	defer span.Finish()
-	ctx = logtags.AddTag(ctx, opName, nil)
-
-	if err := m.server.stopper.RunTaskWithErr(ctx, opName, func(
-		ctx context.Context,
-	) error {
-		// Same as in SyncAllEngines, because stores can be added asynchronously, we
-		// need to ensure that the bootstrap process has happened.
-		m.server.node.waitForAdditionalStoreInit()
-
-		for _, eng := range m.server.engines {
-			if err := eng.SetMinVersion(*req.Version); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	resp := &serverpb.DeprecateBaseEncryptionRegistryResponse{}
-	return resp, nil
-}
-
 // WaitForSpanConfigSubscription implements the MigrationServer interface.
 func (m *migrationServer) WaitForSpanConfigSubscription(
 	ctx context.Context, _ *serverpb.WaitForSpanConfigSubscriptionRequest,

pkg/server/serverpb/migration.proto

@@ -52,16 +52,6 @@ message SyncAllEnginesRequest{}
 // SyncAllEnginesResponse is the response to a SyncAllEnginesRequest.
 message SyncAllEnginesResponse{}
 
-// DeprecateBaseEncryptionRegistryRequest is used to instruct the target node
-// to stop using the Base version monolithic encryption-at-rest registry.
-message DeprecateBaseEncryptionRegistryRequest {
-  roachpb.Version version = 1;
-}
-
-// DeprecateBaseEncryptionRegistryResponse is the response to a
-// DeprecateBaseEncryptionRegistryRequest.
-message DeprecateBaseEncryptionRegistryResponse{}
-
 // WaitForSpanConfigSubscriptionRequest waits until the target node is wholly
 // subscribed to the global span configurations state.
 message WaitForSpanConfigSubscriptionRequest{}
@@ -95,10 +85,6 @@ service Migration {
    // replicas with a version less than the one provided.
    rpc PurgeOutdatedReplicas (PurgeOutdatedReplicasRequest) returns (PurgeOutdatedReplicasResponse) { }
 
-   // DeprecateBaseRegistry is used to instruct the target node to stop
-   // using the Base version monolithic encryption-at-rest registry.
-   rpc DeprecateBaseEncryptionRegistry (DeprecateBaseEncryptionRegistryRequest) returns (DeprecateBaseEncryptionRegistryResponse) { }
-
    // WaitForSpanConfigSubscription waits until the target node is wholly
    // subscribed to the global span configurations state.
    rpc WaitForSpanConfigSubscription (WaitForSpanConfigSubscriptionRequest) returns (WaitForSpanConfigSubscriptionResponse) { }

pkg/sql/catalog/catconstants/constants.go

@@ -385,6 +385,6 @@ var DefaultHashShardedIndexBucketCount = settings.RegisterIntSetting(
 	settings.TenantWritable,
 	"sql.defaults.default_hash_sharded_index_bucket_count",
 	"used as bucket count if bucket count is not specified in hash sharded index definition",
-	8,
+	16,
 	settings.NonNegativeInt,
 ).WithPublic()

pkg/sql/distsql/columnar_operators_test.go

@@ -93,6 +93,11 @@ var aggregateFuncToNumArguments = map[execinfrapb.AggregatorSpec_Func]int{
 	execinfrapb.FinalRegrSxx:            1,
 	execinfrapb.FinalRegrSxy:            1,
 	execinfrapb.FinalRegrSyy:            1,
+	execinfrapb.FinalRegrAvgx:           1,
+	execinfrapb.FinalRegrAvgy:           1,
+	execinfrapb.FinalRegrIntercept:      1,
+	execinfrapb.FinalRegrR2:             1,
+	execinfrapb.FinalRegrSlope:          1,
 }
 
 // TestAggregateFuncToNumArguments ensures that all aggregate functions are

pkg/sql/execinfra/version.go

@@ -39,7 +39,7 @@ import "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb"
 //
 // ATTENTION: When updating these fields, add a brief description of what
 // changed to the version history below.
-const Version execinfrapb.DistSQLVersion = 60
+const Version execinfrapb.DistSQLVersion = 61
 
 // MinAcceptedVersion is the oldest version that the server is compatible with.
 // A server will not accept flows with older versions.
@@ -51,6 +51,14 @@ const MinAcceptedVersion execinfrapb.DistSQLVersion = 60
 
 Please add new entries at the top.
 
+- Version: 61 (MinAcceptedVersion: 60)
+  - final_regr_avgx, final_regr_avgy, final_regr_intercept, final_regr_r2, and
+    final_regr_slope aggregate functions were introduced to support local and
+    final aggregation of the corresponding builtin functions. It would be
+    unrecognized by a server running older versions, hence the version bump.
+    However, a server running v61 can still process all plans from servers
+    running v60, thus the MinAcceptedVersion is kept at 60.
+
 - Version: 60 (MinAcceptedVersion: 60):
  - Deprecated ExportWriterSpec and ParquetWriterSpec and merged them into ExportSpec
 

pkg/sql/execinfrapb/aggregate_funcs.go

@@ -65,4 +65,9 @@ const (
 	FinalRegrSxx            = AggregatorSpec_FINAL_REGR_SXX
 	FinalRegrSxy            = AggregatorSpec_FINAL_REGR_SXY
 	FinalRegrSyy            = AggregatorSpec_FINAL_REGR_SYY
+	FinalRegrAvgx           = AggregatorSpec_FINAL_REGR_AVGX
+	FinalRegrAvgy           = AggregatorSpec_FINAL_REGR_AVGY
+	FinalRegrIntercept      = AggregatorSpec_FINAL_REGR_INTERCEPT
+	FinalRegrR2             = AggregatorSpec_FINAL_REGR_R2
+	FinalRegrSlope          = AggregatorSpec_FINAL_REGR_SLOPE
 )

pkg/sql/execinfrapb/processors_sql.proto

@@ -832,6 +832,11 @@ message AggregatorSpec {
     FINAL_REGR_SXX = 50;
     FINAL_REGR_SXY = 51;
     FINAL_REGR_SYY = 52;
+    FINAL_REGR_AVGX = 53;
+    FINAL_REGR_AVGY = 54;
+    FINAL_REGR_INTERCEPT = 55;
+    FINAL_REGR_R2 = 56;
+    FINAL_REGR_SLOPE = 57;
   }
 
   enum Type {

pkg/sql/logictest/testdata/logic_test/distsql_agg

@@ -622,11 +622,21 @@ SELECT covar_pop(y, x)::decimal FROM statistics_agg_test
 ----
 3.75
 
+query FFF
+SELECT regr_intercept(y, x), regr_r2(y, x), regr_slope(y, x) FROM statistics_agg_test
+----
+48.4545454545455 0.00204565911136568 0.454545454545455
+
 query FFF
 SELECT regr_sxx(y, x), regr_sxy(y, x), regr_syy(y, x) FROM statistics_agg_test
 ----
 825 375 83325
 
+query FF
+SELECT regr_avgx(y, x), regr_avgy(y, x) FROM statistics_agg_test
+----
+4.5 50.5
+
 # Regression test for #37211 (incorrect ordering between aggregator stages).
 statement ok
 CREATE TABLE uv (u INT PRIMARY KEY, v INT);

pkg/sql/logictest/testdata/logic_test/hash_sharded_index

@@ -885,46 +885,46 @@ t_hash_pre_split  142       t_hash_pre_split_idx_b  /Table/142/2/7  /Max
 subtest test_default_bucket_count
 
 statement ok
-CREATE TABLE t_default_bucket_8 (
+CREATE TABLE t_default_bucket_16 (
   a INT PRIMARY KEY USING HASH,
   b INT,
   c INT,
-  INDEX idx_t_default_bucket_8_b (b) USING HASH,
-  INDEX idx_t_default_bucket_8_c (c) USING HASH WITH (bucket_count=4),
+  INDEX idx_t_default_bucket_16_b (b) USING HASH,
+  INDEX idx_t_default_bucket_16_c (c) USING HASH WITH (bucket_count=4),
   FAMILY fam_0_a (a),
   FAMILY fam_1_c_b (c, b)
 );
 
 query T
-SELECT @2 FROM [SHOW CREATE TABLE t_default_bucket_8]
+SELECT @2 FROM [SHOW CREATE TABLE t_default_bucket_16]
 ----
-CREATE TABLE public.t_default_bucket_8 (
-   crdb_internal_a_shard_8 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(a)), 8:::INT8)) VIRTUAL,
+CREATE TABLE public.t_default_bucket_16 (
+   crdb_internal_a_shard_16 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(a)), 16:::INT8)) VIRTUAL,
    a INT8 NOT NULL,
    b INT8 NULL,
    c INT8 NULL,
-   crdb_internal_b_shard_8 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(b)), 8:::INT8)) VIRTUAL,
+   crdb_internal_b_shard_16 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(b)), 16:::INT8)) VIRTUAL,
    crdb_internal_c_shard_4 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(c)), 4:::INT8)) VIRTUAL,
-   CONSTRAINT t_default_bucket_8_pkey PRIMARY KEY (a ASC) USING HASH WITH (bucket_count=8),
-   INDEX idx_t_default_bucket_8_b (b ASC) USING HASH WITH (bucket_count=8),
-   INDEX idx_t_default_bucket_8_c (c ASC) USING HASH WITH (bucket_count=4),
+   CONSTRAINT t_default_bucket_16_pkey PRIMARY KEY (a ASC) USING HASH WITH (bucket_count=16),
+   INDEX idx_t_default_bucket_16_b (b ASC) USING HASH WITH (bucket_count=16),
+   INDEX idx_t_default_bucket_16_c (c ASC) USING HASH WITH (bucket_count=4),
    FAMILY fam_0_a (a),
    FAMILY fam_1_c_b (c, b)
 )
 
 statement ok
-SET CLUSTER SETTING sql.defaults.default_hash_sharded_index_bucket_count = 16
+SET CLUSTER SETTING sql.defaults.default_hash_sharded_index_bucket_count = 8
 
 statement ok
-CREATE TABLE t_default_bucket_16 (a INT PRIMARY KEY USING HASH);
+CREATE TABLE t_default_bucket_8 (a INT PRIMARY KEY USING HASH);
 
 query T
-SELECT @2 FROM [SHOW CREATE TABLE t_default_bucket_16]
+SELECT @2 FROM [SHOW CREATE TABLE t_default_bucket_8]
 ----
-CREATE TABLE public.t_default_bucket_16 (
-   crdb_internal_a_shard_16 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(a)), 16:::INT8)) VIRTUAL,
+CREATE TABLE public.t_default_bucket_8 (
+   crdb_internal_a_shard_8 INT4 NOT VISIBLE NOT NULL AS (mod(fnv32(crdb_internal.datums_to_bytes(a)), 8:::INT8)) VIRTUAL,
    a INT8 NOT NULL,
-   CONSTRAINT t_default_bucket_16_pkey PRIMARY KEY (a ASC) USING HASH WITH (bucket_count=16),
+   CONSTRAINT t_default_bucket_8_pkey PRIMARY KEY (a ASC) USING HASH WITH (bucket_count=8),
    FAMILY "primary" (a)
 )