Merge #76742 #76944

76742: dev: don't exit immediately in ./dev ui watch r=irfansharif a=sjbarag

./dev ui watch previously didn't wait for the command context for webpack commands to complete before returning. This caused the 'watch' subcommand to stop immediately, then the 'ui' subcommand, then the entire 'dev' command to stop, exiting the entire process likely before webpack could even start. Wait for the webpack commands to complete (which they never should, since they're meant to run indefinitely) before returning from ./dev ui watch's implementation.

Release note: None

76944: *: enable declarative schema changer r=postamar a=postamar

This change enables the declarative schema changer by default. This PR originated as #76610 by `@fqazi` which I rebased following the merging of #76776. Only mild alterations were needed, these commits can therefore be considered as having been reviewed. 

Co-authored-by: Sean Barag <[email protected]>
Co-authored-by: Andrew Werner <[email protected]>
Co-authored-by: Faizan Qazi <[email protected]>
Co-authored-by: Marius Posta <[email protected]>

Makefile

@@ -130,7 +130,7 @@ SUBTESTS :=
 LINTTIMEOUT := 30m
 
 ## Test timeout to use for regular tests.
-TESTTIMEOUT := 45m
+TESTTIMEOUT := 60m
 
 ## Test timeout to use for race tests.
 RACETIMEOUT := 45m

dev

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 # Bump this counter to force rebuilding `dev` on all machines.
-DEV_VERSION=17
+DEV_VERSION=18
 
 THIS_DIR=$(cd "$(dirname "$0")" && pwd)
 BINARY_DIR=$THIS_DIR/bin/dev-versions

docs/generated/settings/settings-for-tenants.txt

@@ -96,7 +96,6 @@ sql.defaults.experimental_auto_rehoming.enabled	boolean	false	default value for
 sql.defaults.experimental_distsql_planning	enumeration	off	default experimental_distsql_planning mode; enables experimental opt-driven DistSQL planning [off = 0, on = 1]
 sql.defaults.experimental_enable_unique_without_index_constraints.enabled	boolean	false	default value for experimental_enable_unique_without_index_constraints session setting;disables unique without index constraints by default
 sql.defaults.experimental_implicit_column_partitioning.enabled	boolean	false	default value for experimental_enable_temp_tables; allows for the use of implicit column partitioning
-sql.defaults.experimental_new_schema_changer.enabled	enumeration	off	default value for experimental_use_new_schema_changer session setting;disables new schema changer by default [off = 0, on = 1, unsafe = 2, unsafe_always = 3]
 sql.defaults.experimental_stream_replication.enabled	boolean	false	default value for experimental_stream_replication session setting;enables the ability to setup a replication stream
 sql.defaults.experimental_temporary_tables.enabled	boolean	false	default value for experimental_enable_temp_tables; allows for use of temporary tables by default
 sql.defaults.foreign_key_cascades_limit	integer	10000	default value for foreign_key_cascades_limit session setting; limits the number of cascading operations that run as part of a single query
@@ -125,6 +124,7 @@ sql.defaults.transaction_rows_read_err	integer	0	the limit for the number of row
 sql.defaults.transaction_rows_read_log	integer	0	the threshold for the number of rows read by a SQL transaction which - once exceeded - will trigger a logging event to SQL_PERF (or SQL_INTERNAL_PERF for internal transactions); use 0 to disable
 sql.defaults.transaction_rows_written_err	integer	0	the limit for the number of rows written by a SQL transaction which - once exceeded - will fail the transaction (or will trigger a logging event to SQL_INTERNAL_PERF for internal transactions); use 0 to disable
 sql.defaults.transaction_rows_written_log	integer	0	the threshold for the number of rows written by a SQL transaction which - once exceeded - will trigger a logging event to SQL_PERF (or SQL_INTERNAL_PERF for internal transactions); use 0 to disable
+sql.defaults.use_declarative_schema_changer	enumeration	on	default value for use_declarative_schema_changer session setting;disables new schema changer by default [off = 0, on = 1, unsafe = 2, unsafe_always = 3]
 sql.defaults.vectorize	enumeration	on	default vectorize mode [on = 0, on = 2, experimental_always = 3, off = 4]
 sql.defaults.zigzag_join.enabled	boolean	true	default value for enable_zigzag_join session setting; allows use of zig-zag join by default
 sql.distsql.max_running_flows	integer	-128	the value - when positive - used as is, or the value - when negative - multiplied by the number of CPUs on a node, to determine the maximum number of concurrent remote flows that can be run on the node

docs/generated/settings/settings.html

@@ -108,7 +108,6 @@
 <tr><td><code>sql.defaults.experimental_distsql_planning</code></td><td>enumeration</td><td><code>off</code></td><td>default experimental_distsql_planning mode; enables experimental opt-driven DistSQL planning [off = 0, on = 1]</td></tr>
 <tr><td><code>sql.defaults.experimental_enable_unique_without_index_constraints.enabled</code></td><td>boolean</td><td><code>false</code></td><td>default value for experimental_enable_unique_without_index_constraints session setting;disables unique without index constraints by default</td></tr>
 <tr><td><code>sql.defaults.experimental_implicit_column_partitioning.enabled</code></td><td>boolean</td><td><code>false</code></td><td>default value for experimental_enable_temp_tables; allows for the use of implicit column partitioning</td></tr>
-<tr><td><code>sql.defaults.experimental_new_schema_changer.enabled</code></td><td>enumeration</td><td><code>off</code></td><td>default value for experimental_use_new_schema_changer session setting;disables new schema changer by default [off = 0, on = 1, unsafe = 2, unsafe_always = 3]</td></tr>
 <tr><td><code>sql.defaults.experimental_stream_replication.enabled</code></td><td>boolean</td><td><code>false</code></td><td>default value for experimental_stream_replication session setting;enables the ability to setup a replication stream</td></tr>
 <tr><td><code>sql.defaults.experimental_temporary_tables.enabled</code></td><td>boolean</td><td><code>false</code></td><td>default value for experimental_enable_temp_tables; allows for use of temporary tables by default</td></tr>
 <tr><td><code>sql.defaults.foreign_key_cascades_limit</code></td><td>integer</td><td><code>10000</code></td><td>default value for foreign_key_cascades_limit session setting; limits the number of cascading operations that run as part of a single query</td></tr>
@@ -137,6 +136,7 @@
 <tr><td><code>sql.defaults.transaction_rows_read_log</code></td><td>integer</td><td><code>0</code></td><td>the threshold for the number of rows read by a SQL transaction which - once exceeded - will trigger a logging event to SQL_PERF (or SQL_INTERNAL_PERF for internal transactions); use 0 to disable</td></tr>
 <tr><td><code>sql.defaults.transaction_rows_written_err</code></td><td>integer</td><td><code>0</code></td><td>the limit for the number of rows written by a SQL transaction which - once exceeded - will fail the transaction (or will trigger a logging event to SQL_INTERNAL_PERF for internal transactions); use 0 to disable</td></tr>
 <tr><td><code>sql.defaults.transaction_rows_written_log</code></td><td>integer</td><td><code>0</code></td><td>the threshold for the number of rows written by a SQL transaction which - once exceeded - will trigger a logging event to SQL_PERF (or SQL_INTERNAL_PERF for internal transactions); use 0 to disable</td></tr>
+<tr><td><code>sql.defaults.use_declarative_schema_changer</code></td><td>enumeration</td><td><code>on</code></td><td>default value for use_declarative_schema_changer session setting;disables new schema changer by default [off = 0, on = 1, unsafe = 2, unsafe_always = 3]</td></tr>
 <tr><td><code>sql.defaults.vectorize</code></td><td>enumeration</td><td><code>on</code></td><td>default vectorize mode [on = 0, on = 2, experimental_always = 3, off = 4]</td></tr>
 <tr><td><code>sql.defaults.zigzag_join.enabled</code></td><td>boolean</td><td><code>true</code></td><td>default value for enable_zigzag_join session setting; allows use of zig-zag join by default</td></tr>
 <tr><td><code>sql.distsql.max_running_flows</code></td><td>integer</td><td><code>-128</code></td><td>the value - when positive - used as is, or the value - when negative - multiplied by the number of CPUs on a node, to determine the maximum number of concurrent remote flows that can be run on the node</td></tr>

pkg/bench/rttanalysis/testdata/benchmark_expectations

@@ -1,52 +1,52 @@
 exp,benchmark
-15,AlterRole/alter_role_with_1_option
-16,AlterRole/alter_role_with_2_options
-21,AlterRole/alter_role_with_3_options
-18-19,AlterTableAddCheckConstraint/alter_table_add_1_check_constraint
-18-19,AlterTableAddCheckConstraint/alter_table_add_2_check_constraints
-18-19,AlterTableAddCheckConstraint/alter_table_add_3_check_constraints
+14,AlterRole/alter_role_with_1_option
+15,AlterRole/alter_role_with_2_options
+20,AlterRole/alter_role_with_3_options
+18,AlterTableAddCheckConstraint/alter_table_add_1_check_constraint
+18,AlterTableAddCheckConstraint/alter_table_add_2_check_constraints
+18,AlterTableAddCheckConstraint/alter_table_add_3_check_constraints
 18,AlterTableAddColumn/alter_table_add_1_column
 18,AlterTableAddColumn/alter_table_add_2_columns
 18,AlterTableAddColumn/alter_table_add_3_columns
 23,AlterTableAddForeignKey/alter_table_add_1_foreign_key
 28,AlterTableAddForeignKey/alter_table_add_2_foreign_keys
 33,AlterTableAddForeignKey/alter_table_add_3_foreign_keys
 23,AlterTableAddForeignKey/alter_table_add_foreign_key_with_3_columns
-24,AlterTableConfigureZone/alter_table_configure_zone_5_replicas
-24,AlterTableConfigureZone/alter_table_configure_zone_7_replicas_
-24,AlterTableConfigureZone/alter_table_configure_zone_ranges
-20,AlterTableDropColumn/alter_table_drop_1_column
-21,AlterTableDropColumn/alter_table_drop_2_columns
-22,AlterTableDropColumn/alter_table_drop_3_columns
+23,AlterTableConfigureZone/alter_table_configure_zone_5_replicas
+23,AlterTableConfigureZone/alter_table_configure_zone_7_replicas_
+23,AlterTableConfigureZone/alter_table_configure_zone_ranges
+19,AlterTableDropColumn/alter_table_drop_1_column
+20,AlterTableDropColumn/alter_table_drop_2_columns
+21,AlterTableDropColumn/alter_table_drop_3_columns
 18,AlterTableDropConstraint/alter_table_drop_1_check_constraint
 18,AlterTableDropConstraint/alter_table_drop_2_check_constraints
 18,AlterTableDropConstraint/alter_table_drop_3_check_constraints
-15-16,AlterTableSplit/alter_table_split_at_1_value
-21-22,AlterTableSplit/alter_table_split_at_2_values
-27-28,AlterTableSplit/alter_table_split_at_3_values
+15,AlterTableSplit/alter_table_split_at_1_value
+21,AlterTableSplit/alter_table_split_at_2_values
+27,AlterTableSplit/alter_table_split_at_3_values
 11,AlterTableUnsplit/alter_table_unsplit_at_1_value
 13,AlterTableUnsplit/alter_table_unsplit_at_2_values
 15,AlterTableUnsplit/alter_table_unsplit_at_3_values
 17,CreateRole/create_role_with_1_option
 19,CreateRole/create_role_with_2_options
 20,CreateRole/create_role_with_3_options
-18-19,CreateRole/create_role_with_no_options
-22,DropDatabase/drop_database_0_tables
-28,DropDatabase/drop_database_1_table
-34,DropDatabase/drop_database_2_tables
-40,DropDatabase/drop_database_3_tables
-25,DropRole/drop_1_role
-32,DropRole/drop_2_roles
-39,DropRole/drop_3_roles
-19,DropSequence/drop_1_sequence
-25,DropSequence/drop_2_sequences
-31,DropSequence/drop_3_sequences
-20-24,DropTable/drop_1_table
-27-31,DropTable/drop_2_tables
-34-38,DropTable/drop_3_tables
-22-25,DropView/drop_1_view
-29-32,DropView/drop_2_views
-36,DropView/drop_3_views
+18,CreateRole/create_role_with_no_options
+20,DropDatabase/drop_database_0_tables
+22,DropDatabase/drop_database_1_table
+24,DropDatabase/drop_database_2_tables
+26,DropDatabase/drop_database_3_tables
+24,DropRole/drop_1_role
+31,DropRole/drop_2_roles
+38,DropRole/drop_3_roles
+20,DropSequence/drop_1_sequence
+23,DropSequence/drop_2_sequences
+26,DropSequence/drop_3_sequences
+20,DropTable/drop_1_table
+23,DropTable/drop_2_tables
+26,DropTable/drop_3_tables
+23,DropView/drop_1_view
+26,DropView/drop_2_views
+29,DropView/drop_3_views
 19,Grant/grant_all_on_1_table
 19,Grant/grant_all_on_2_tables
 19,Grant/grant_all_on_3_tables
@@ -84,11 +84,11 @@ exp,benchmark
 1,SystemDatabaseQueries/select_system.users_with_empty_database_Name
 1,SystemDatabaseQueries/select_system.users_with_schema_Name
 2,SystemDatabaseQueries/select_system.users_without_schema_Name
-23-24,Truncate/truncate_1_column_0_rows
-23-24,Truncate/truncate_1_column_1_row
-23-24,Truncate/truncate_1_column_2_rows
-23-24,Truncate/truncate_2_column_0_rows
-23-24,Truncate/truncate_2_column_1_rows
-24,Truncate/truncate_2_column_2_rows
+23,Truncate/truncate_1_column_0_rows
+23,Truncate/truncate_1_column_1_row
+23,Truncate/truncate_1_column_2_rows
+23,Truncate/truncate_2_column_0_rows
+23,Truncate/truncate_2_column_1_rows
+23,Truncate/truncate_2_column_2_rows
 1,VirtualTableQueries/select_crdb_internal.invalid_objects_with_1_fk
 1,VirtualTableQueries/select_crdb_internal.tables_with_1_fk

pkg/ccl/backupccl/testdata/backup-restore/backup-dropped-descriptors

@@ -6,6 +6,7 @@ new-server name=s1
 ----
 
 exec-sql
+SET use_declarative_schema_changer = 'off';
 SET CLUSTER SETTING jobs.debug.pausepoints = 'schemachanger.before.exec';
 CREATE DATABASE d;
 CREATE TABLE d.foo (id INT);
@@ -101,6 +102,7 @@ exec-sql
 CREATE TYPE d2.typ AS ENUM ('hello');
 CREATE SCHEMA d2.s;
 CREATE TABLE d2.s.t (id INT);
+SET use_declarative_schema_changer = 'off';
 SET CLUSTER SETTING jobs.debug.pausepoints = 'schemachanger.before.exec';
 DROP SCHEMA d2.s CASCADE;
 ----

pkg/ccl/logictestccl/testdata/logic_test/new_schema_changer

@@ -1,5 +1,5 @@
 statement ok
-SET experimental_use_new_schema_changer = 'unsafe'
+SET use_declarative_schema_changer = 'unsafe'
 
 subtest create_index
 

pkg/ccl/partitionccl/drop_test.go

@@ -262,6 +262,6 @@ SELECT job_id
 
 		waitForJobDone(t, tdb, "GC for DROP INDEX%idx")
 		tdb.Exec(t, `ALTER RANGE default CONFIGURE ZONE USING gc.ttlseconds = 1`)
-		waitForJobDone(t, tdb, "GC for DROP TABLE%t")
+		waitForJobDone(t, tdb, "GC for dropping descriptor %")
 	})
 }

pkg/ccl/schemachangerccl/BUILD.bazel

@@ -4,7 +4,7 @@ go_test(
     name = "schemachangerccl_test",
     srcs = [
         "main_test.go",
-        "schemachanger_end_to_end_test.go",
+        "schemachanger_ccl_test.go",
     ],
     data = glob(["testdata/**"]) + [
         "//pkg/sql/schemachanger:testdata",

pkg/ccl/schemachangerccl/schemachanger_end_to_end_test.go → pkg/ccl/schemachangerccl/schemachanger_ccl_test.go

@@ -45,17 +45,21 @@ func sharedTestdata(t *testing.T) string {
 	return testdataDir
 }
 
+func endToEndPath(t *testing.T) string {
+	return testutils.TestDataPath(t, "end_to_end")
+}
+
 func TestSchemaChangerSideEffects(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	sctest.EndToEndSideEffects(t, testutils.TestDataPath(t), newCluster)
+	sctest.EndToEndSideEffects(t, endToEndPath(t), newCluster)
 }
 
 func TestBackupRestore(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 	t.Run("ccl", func(t *testing.T) {
-		sctest.Backup(t, testutils.TestDataPath(t), newCluster)
+		sctest.Backup(t, endToEndPath(t), newCluster)
 	})
 	t.Run("non-ccl", func(t *testing.T) {
 		sctest.Backup(t, sharedTestdata(t), sctest.SingleNodeCluster)
@@ -66,12 +70,19 @@ func TestRollback(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
-	sctest.Rollback(t, testutils.TestDataPath(t), newCluster)
+	sctest.Rollback(t, endToEndPath(t), newCluster)
 }
 
 func TestPause(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
-	sctest.Pause(t, testutils.TestDataPath(t), newCluster)
+	sctest.Pause(t, endToEndPath(t), newCluster)
+}
+
+func TestDecomposeToElements(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	sctest.DecomposeToElements(t, testutils.TestDataPath(t, "decomp"), newCluster)
 }