Merge cockroachdb#97501

97501: sql: use udf in column default value in new table r=chengxiong-ruan a=chengxiong-ruan

Informs: cockroachdb#87699

**sql: split ColumnDefaultExpr context into more granular ones**
Previously we only have one `ColumnDefaultExpr` context. This is
not enough for the granularity we want to control over UDF use case
allowance because we want to allow UDF in column default for new
tables but not existing tables. We also want allow UDF for
`SET DEFAULT`. To address, this commit split the context into more
granular ones so that we can do better allowance control.


**sql: use UDFs in column DEFAULT of new tables and SET DEFAULT**
Release note (sql change): Previously, we didn't allow using
UDFs in column DEFAULT. This commit adds support for that but
only allows UDFs in column DEFAULT when creating a new table or
doing SET DEFAULT.


**sql: add test cases of renaming a function used in CHECK**
Release note: None


**sql: fix backup and restore to work with UDFs in column DEFAULT**
Release note (enterprise change): previously UDFs can't be referenced
from column DEFAULTs. Now we turned on UDF's usage in DEFAULTs, we need
to teach BACKUP and RESTORE to rewrite UDF ids referenced in column's
DEFAULT expressions. If UDF dependencies are missing and `skip_missing_udfs`
flag is given, the DEFAULT expressions would be dropped during the RESTORE.



Co-authored-by: Chengxiong Ruan <[email protected]>

pkg/ccl/backupccl/testdata/backup-restore/user-defined-functions-in-checks

@@ -111,9 +111,14 @@ USE db1
 
 # Make sure function ids in CHECK constraint are rewritten.
 query-sql
-SELECT create_statement FROM [SHOW CREATE TABLE sc1.f1]
+SELECT create_statement FROM [SHOW CREATE TABLE sc1.t1]
 ----
-pq: relation "sc1.f1" does not exist
+CREATE TABLE sc1.t1 (
+	a INT8 NOT NULL,
+	b INT8 NULL,
+	CONSTRAINT t1_pkey PRIMARY KEY (a ASC),
+	CONSTRAINT check_b CHECK (sc1.f1(b) > 1:::INT8)
+)
 
 # Make sure that CHECK constraint still applies correctly
 query-sql

pkg/ccl/backupccl/testdata/backup-restore/user-defined-functions-in-defaults

@@ -0,0 +1,235 @@
+# Test backing up and restoring a database with user defined functions.
+new-cluster name=s
+----
+
+exec-sql
+CREATE DATABASE db1;
+USE db1;
+CREATE SCHEMA sc1;
+CREATE FUNCTION sc1.f1() RETURNS INT LANGUAGE SQL AS $$ SELECT 1 $$;
+CREATE TABLE sc1.t1(a INT PRIMARY KEY, b INT DEFAULT sc1.f1());
+----
+
+exec-sql
+BACKUP DATABASE db1 INTO 'nodelocal://0/test/'
+----
+
+query-sql
+WITH descs AS (
+  SHOW BACKUP LATEST IN 'nodelocal://0/test/'
+)
+SELECT database_name, parent_schema_name, object_name, object_type, is_full_cluster FROM descs
+----
+<nil> <nil> db1 database false
+db1 <nil> public schema false
+db1 <nil> sc1 schema false
+db1 sc1 f1 function false
+db1 sc1 t1 table false
+
+exec-sql
+RESTORE DATABASE db1 FROM LATEST IN 'nodelocal://0/test/' WITH new_db_name = db1_new
+----
+
+exec-sql
+USE db1_new
+----
+
+# Make sure function ids in DEFAULT are rewritten.
+query-sql
+SELECT create_statement FROM [SHOW CREATE TABLE sc1.t1]
+----
+CREATE TABLE sc1.t1 (
+	a INT8 NOT NULL,
+	b INT8 NULL DEFAULT sc1.f1(),
+	CONSTRAINT t1_pkey PRIMARY KEY (a ASC)
+)
+
+# Make sure that the DEFAULT expression still works.
+query-sql
+INSERT INTO sc1.t1 VALUES (1), (2);
+----
+
+query-sql
+SELECT * FROM sc1.t1 ORDER BY a;
+----
+1 1
+2 1
+
+# Make sure that depenency IDs are rewritten by checking DROP FUNCTION errors.
+# Note that technically this only tests forward-reference IDs in depended-on
+# objects are rewritten. But since we have cross-references validation, so this
+# also means back-references in UDF descriptor are good.
+query-sql
+DROP FUNCTION sc1.f1;
+----
+pq: cannot drop function "f1" because other objects ([db1_new.sc1.t1]) still depend on it
+
+# Test backing up and restoring a full cluster with user defined function.
+new-cluster name=s1
+----
+
+exec-sql cluster=s1
+CREATE DATABASE db1;
+USE db1;
+CREATE SCHEMA sc1;
+CREATE FUNCTION sc1.f1() RETURNS INT LANGUAGE SQL AS $$ SELECT 1 $$;
+CREATE TABLE sc1.t1(a INT PRIMARY KEY, b INT DEFAULT sc1.f1());
+----
+
+exec-sql
+BACKUP INTO 'nodelocal://0/test/'
+----
+
+query-sql
+WITH descs AS (
+  SHOW BACKUP LATEST IN 'nodelocal://0/test/'
+)
+SELECT
+  database_name, parent_schema_name, object_name, object_type, is_full_cluster
+FROM
+  descs
+WHERE
+  database_name = 'db1'
+----
+db1 <nil> public schema true
+db1 <nil> sc1 schema true
+db1 sc1 f1 function true
+db1 sc1 t1 table true
+
+# Start a new cluster with the same IO dir.
+new-cluster name=s2 share-io-dir=s1
+----
+
+# Restore into the new cluster.
+exec-sql cluster=s2
+RESTORE FROM LATEST IN 'nodelocal://0/test/'
+----
+
+exec-sql
+USE db1
+----
+
+# Make sure function ids in DEFAULT are rewritten.
+query-sql
+SELECT create_statement FROM [SHOW CREATE TABLE sc1.t1]
+----
+CREATE TABLE sc1.t1 (
+	a INT8 NOT NULL,
+	b INT8 NULL DEFAULT sc1.f1(),
+	CONSTRAINT t1_pkey PRIMARY KEY (a ASC)
+)
+
+# Make sure that the DEFAULT expression still works.
+query-sql
+INSERT INTO sc1.t1 VALUES (1), (2);
+----
+
+query-sql
+SELECT * FROM sc1.t1 ORDER BY a;
+----
+1 1
+2 1
+
+# Make sure that depenency IDs are rewritten by checking DROP FUNCTION errors.
+# Note that technically this only tests forward-reference IDs in depended-on
+# objects are rewritten. But since we have cross-references validation, so this
+# also means back-references in UDF descriptor are good.
+query-sql
+DROP FUNCTION sc1.f1;
+----
+pq: cannot drop function "f1" because other objects ([db1.sc1.t1]) still depend on it
+
+# Make sure that backup and restore individual tables referencing UDFs able to
+# drop DEFAULT.
+new-cluster name=s3
+----
+
+exec-sql cluster=s3
+CREATE DATABASE db1;
+CREATE DATABASE db2;
+CREATE DATABASE db3;
+USE db1;
+CREATE SCHEMA sc1;
+CREATE FUNCTION sc1.f1() RETURNS INT LANGUAGE SQL AS $$ SELECT 1 $$;
+CREATE TABLE sc1.t1(a INT PRIMARY KEY, b INT DEFAULT sc1.f1());
+----
+
+exec-sql
+BACKUP DATABASE db1 INTO 'nodelocal://0/test/'
+----
+
+query-sql
+WITH descs AS (
+  SHOW BACKUP LATEST IN 'nodelocal://0/test/'
+)
+SELECT database_name, parent_schema_name, object_name, object_type, is_full_cluster FROM descs
+----
+<nil> <nil> db1 database false
+db1 <nil> public schema false
+db1 <nil> sc1 schema false
+db1 sc1 f1 function false
+db1 sc1 t1 table false
+
+exec-sql
+RESTORE TABLE sc1.t1 FROM LATEST IN 'nodelocal://0/test/' WITH into_db = 'db2';
+----
+pq: cannot restore table "t1" without referenced function 114 (or "skip_missing_udfs" option)
+
+exec-sql
+RESTORE TABLE sc1.t1 FROM LATEST IN 'nodelocal://0/test/' WITH into_db = 'db2', skip_missing_udfs;
+----
+
+exec-sql
+USE db2
+----
+
+# Make sure DEFAULT is dropped.
+query-sql
+SELECT create_statement FROM [SHOW CREATE TABLE sc1.t1]
+----
+CREATE TABLE sc1.t1 (
+	a INT8 NOT NULL,
+	b INT8 NULL,
+	CONSTRAINT t1_pkey PRIMARY KEY (a ASC)
+)
+
+exec-sql
+USE db1
+----
+
+exec-sql
+BACKUP TABLE sc1.t1 INTO 'nodelocal://0/test/'
+----
+
+query-sql
+WITH descs AS (
+  SHOW BACKUP LATEST IN 'nodelocal://0/test/'
+)
+SELECT database_name, parent_schema_name, object_name, object_type, is_full_cluster FROM descs
+----
+<nil> <nil> db1 database false
+db1 <nil> sc1 schema false
+db1 sc1 t1 table false
+
+exec-sql
+RESTORE TABLE sc1.t1 FROM LATEST IN 'nodelocal://0/test/' WITH into_db = 'db3';
+----
+pq: cannot restore table "t1" without referenced function 114 (or "skip_missing_udfs" option)
+
+exec-sql
+RESTORE TABLE sc1.t1 FROM LATEST IN 'nodelocal://0/test/' WITH into_db = 'db3', skip_missing_udfs;
+----
+
+exec-sql
+USE db3
+----
+
+# Make sure DEFAULT is dropped.
+query-sql
+SELECT create_statement FROM [SHOW CREATE TABLE sc1.t1]
+----
+CREATE TABLE sc1.t1 (
+	a INT8 NOT NULL,
+	b INT8 NULL,
+	CONSTRAINT t1_pkey PRIMARY KEY (a ASC)
+)

pkg/ccl/changefeedccl/avro_test.go

@@ -177,7 +177,7 @@ func avroFieldMetadataToColDesc(
 	def := parsed.AST.(*tree.AlterTable).Cmds[0].(*tree.AlterTableAddColumn).ColumnDef
 	ctx := context.Background()
 	semaCtx := makeTestSemaCtx()
-	cdd, err := tabledesc.MakeColumnDefDescs(ctx, def, &semaCtx, evalCtx)
+	cdd, err := tabledesc.MakeColumnDefDescs(ctx, def, &semaCtx, evalCtx, tree.ColumnDefaultExprInAddColumn)
 	if err != nil {
 		return nil, err
 	}

pkg/ccl/changefeedccl/cdceval/cdc_prev.go

@@ -232,6 +232,14 @@ func (c *prevCol) NumOwnsSequences() int {
 	return 0
 }
 
+func (c *prevCol) NumUsesFunctions() int {
+	return 0
+}
+
+func (c *prevCol) GetUsesFunctionID(ordinal int) descpb.ID {
+	return 0
+}
+
 func (c *prevCol) GetOwnsSequenceID(ownsSequenceOrdinal int) descpb.ID {
 	return 0
 }

pkg/sql/add_column.go

@@ -72,7 +72,7 @@ func (p *planner) addColumnImpl(
 	}
 	d = newDef
 
-	cdd, err := tabledesc.MakeColumnDefDescs(params.ctx, d, &params.p.semaCtx, params.EvalContext())
+	cdd, err := tabledesc.MakeColumnDefDescs(params.ctx, d, &params.p.semaCtx, params.EvalContext(), tree.ColumnDefaultExprInAddColumn)
 	if err != nil {
 		return err
 	}