From 4b4f549432cd1f9de32b2a97ad4d8c47ab1242b1 Mon Sep 17 00:00:00 2001 From: Rohan Yadav Date: Tue, 18 Aug 2020 15:49:35 -0400 Subject: [PATCH] sql: introduce a command to convert a database into a schema Fixes #50885. This commit introduces a command to convert a database into a user defined schema under a desired database. This command can be used by users who are currently emulating a Postgres style set of schemas in CockroachDB with separate databases. Release note (sql change): Users can now convert existing databases into schemas under other databases through the `ALTER DATABASE ... CONVERT TO SCHEMA UNDER PARENT ...` command. This command can only be run by `admin` and is only valid for databases that don't already have any child schemas other than `public`. --- docs/generated/sql/bnf/stmt_block.bnf | 5 + pkg/sql/create_schema.go | 3 +- pkg/sql/drop_cascade.go | 2 +- .../testdata/logic_test/reparent_database | 66 +++++ pkg/sql/opaque.go | 3 + pkg/sql/parser/sql.y | 13 +- pkg/sql/plan.go | 2 + pkg/sql/reparent_database.go | 237 ++++++++++++++++++ pkg/sql/sem/tree/rename.go | 14 ++ pkg/sql/sem/tree/stmt.go | 7 + pkg/sql/walk.go | 1 + 11 files changed, 348 insertions(+), 5 deletions(-) create mode 100644 pkg/sql/logictest/testdata/logic_test/reparent_database create mode 100644 pkg/sql/reparent_database.go diff --git a/docs/generated/sql/bnf/stmt_block.bnf b/docs/generated/sql/bnf/stmt_block.bnf index a1994f0bcb9a..a70c6ab15831 100644 --- a/docs/generated/sql/bnf/stmt_block.bnf +++ b/docs/generated/sql/bnf/stmt_block.bnf @@ -739,6 +739,7 @@ unreserved_keyword ::= | 'CONFIGURE' | 'CONSTRAINTS' | 'CONVERSION' + | 'CONVERT' | 'COPY' | 'COVERING' | 'CREATEROLE' @@ -1111,6 +1112,7 @@ alter_sequence_stmt ::= alter_database_stmt ::= alter_rename_database_stmt | alter_zone_database_stmt + | alter_database_to_schema_stmt alter_range_stmt ::= alter_zone_range_stmt @@ -1522,6 +1524,9 @@ alter_rename_database_stmt ::= alter_zone_database_stmt ::= 'ALTER' 'DATABASE' database_name set_zone_config +alter_database_to_schema_stmt ::= + 'ALTER' 'DATABASE' database_name 'CONVERT' 'TO' 'SCHEMA' 'WITH' 'PARENT' database_name + alter_zone_range_stmt ::= 'ALTER' 'RANGE' zone_name set_zone_config diff --git a/pkg/sql/create_schema.go b/pkg/sql/create_schema.go index 347057260727..49286c738e90 100644 --- a/pkg/sql/create_schema.go +++ b/pkg/sql/create_schema.go @@ -126,8 +126,7 @@ func (*createSchemaNode) Next(runParams) (bool, error) { return false, nil } func (*createSchemaNode) Values() tree.Datums { return tree.Datums{} } func (n *createSchemaNode) Close(ctx context.Context) {} -// CreateSchema creates a schema. Currently only works in IF NOT EXISTS mode, -// for schemas that do in fact already exist. +// CreateSchema creates a schema. func (p *planner) CreateSchema(ctx context.Context, n *tree.CreateSchema) (planNode, error) { return &createSchemaNode{ n: n, diff --git a/pkg/sql/drop_cascade.go b/pkg/sql/drop_cascade.go index d7db404fe061..4643099ff3c0 100644 --- a/pkg/sql/drop_cascade.go +++ b/pkg/sql/drop_cascade.go @@ -73,7 +73,7 @@ func (d *dropCascadeState) resolveCollectedObjects( ctx, tree.ObjectLookupFlags{ // Note we set required to be false here in order to not error out - // if we don't find the object, + // if we don't find the object. CommonLookupFlags: tree.CommonLookupFlags{Required: false}, RequireMutable: true, IncludeOffline: true, diff --git a/pkg/sql/logictest/testdata/logic_test/reparent_database b/pkg/sql/logictest/testdata/logic_test/reparent_database new file mode 100644 index 000000000000..01ef7ae71873 --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/reparent_database @@ -0,0 +1,66 @@ +statement ok +SET experimental_enable_user_defined_schemas = true; +SET experimental_enable_enums = true; + +statement ok +CREATE DATABASE pgdatabase; +CREATE TABLE pgdatabase.t1 (x INT PRIMARY KEY); +CREATE DATABASE pgschema; +CREATE TABLE pgschema.t1 (x INT); +CREATE TABLE pgschema.t2 (x INT); +CREATE TABLE pgschema.t3 (x INT PRIMARY KEY); +ALTER TABLE pgschema.t3 ADD FOREIGN KEY (x) REFERENCES pgdatabase.t1 (x); -- references shouldn't be affected by reparenting. +CREATE TYPE pgschema.typ AS ENUM ('schema'); + +let $db_id +SELECT id FROM system.namespace WHERE name = 'pgschema' + +statement ok +ALTER DATABASE pgschema CONVERT TO SCHEMA WITH PARENT pgdatabase + +query I +SELECT * FROM pgdatabase.pgschema.t1 + +query I +SELECT * FROM pgdatabase.pgschema.t2 + +query T +SELECT 'schema'::pgdatabase.pgschema.typ +---- +schema + +statement error pq: insert on table "t3" violates foreign key constraint "fk_x_ref_t1" +INSERT INTO pgdatabase.pgschema.t3 VALUES (1) + +# Assert there aren't any namespace entries left with the old parentID. +query T +SELECT name FROM system.namespace WHERE "parentID" = $db_id + +# We can't reparent a database that has any child schemas. +statement ok +CREATE DATABASE parent; +USE parent; +CREATE SCHEMA child; +USE test; + +statement error pq: cannot convert database with schemas into schema +ALTER DATABASE parent CONVERT TO SCHEMA WITH PARENT pgdatabase + +# We can't reparent a database if it causes a name conflict. +statement ok +CREATE DATABASE pgschema + +statement error pq: schema "pgschema" already exists +ALTER DATABASE pgschema CONVERT TO SCHEMA WITH PARENT pgdatabase + +statement ok +DROP DATABASE pgschema + +# We can't convert a database with an invalid schema name into a schema. +statement ok +CREATE DATABASE pg_temp + +statement error pq: unacceptable schema name "pg_temp" +ALTER DATABASE pg_temp CONVERT TO SCHEMA WITH PARENT pgdatabase + + diff --git a/pkg/sql/opaque.go b/pkg/sql/opaque.go index 7f700b08b042..8400679edb5e 100644 --- a/pkg/sql/opaque.go +++ b/pkg/sql/opaque.go @@ -117,6 +117,8 @@ func buildOpaque( plan, err = p.RenameColumn(ctx, n) case *tree.RenameDatabase: plan, err = p.RenameDatabase(ctx, n) + case *tree.ReparentDatabase: + plan, err = p.ReparentDatabase(ctx, n) case *tree.RenameIndex: plan, err = p.RenameIndex(ctx, n) case *tree.RenameTable: @@ -213,6 +215,7 @@ func init() { &tree.RenameDatabase{}, &tree.RenameIndex{}, &tree.RenameTable{}, + &tree.ReparentDatabase{}, &tree.Revoke{}, &tree.RevokeRole{}, &tree.Scatter{}, diff --git a/pkg/sql/parser/sql.y b/pkg/sql/parser/sql.y index ae181622abb9..4e369614f55c 100644 --- a/pkg/sql/parser/sql.y +++ b/pkg/sql/parser/sql.y @@ -576,7 +576,7 @@ func (u *sqlSymUnion) executorType() tree.ScheduledJobExecutorType { %token CHARACTER CHARACTERISTICS CHECK CLOSE %token CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT %token COMMITTED COMPACT COMPLETE CONCAT CONCURRENTLY CONFIGURATION CONFIGURATIONS CONFIGURE -%token CONFLICT CONSTRAINT CONSTRAINTS CONTAINS CONVERSION COPY COVERING CREATE CREATEROLE +%token CONFLICT CONSTRAINT CONSTRAINTS CONTAINS CONVERSION CONVERT COPY COVERING CREATE CREATEROLE %token CROSS CUBE CURRENT CURRENT_CATALOG CURRENT_DATE CURRENT_SCHEMA %token CURRENT_ROLE CURRENT_TIME CURRENT_TIMESTAMP %token CURRENT_USER CYCLE @@ -714,6 +714,7 @@ func (u *sqlSymUnion) executorType() tree.ScheduledJobExecutorType { // ALTER DATABASE %type alter_rename_database_stmt +%type alter_database_to_schema_stmt %type alter_zone_database_stmt // ALTER INDEX @@ -1383,7 +1384,8 @@ alter_sequence_options_stmt: // %SeeAlso: WEBDOCS/alter-database.html alter_database_stmt: alter_rename_database_stmt -| alter_zone_database_stmt +| alter_zone_database_stmt +| alter_database_to_schema_stmt // ALTER DATABASE has its error help token here because the ALTER DATABASE // prefix is spread over multiple non-terminals. | ALTER DATABASE error // SHOW HELP: ALTER DATABASE @@ -6564,6 +6566,12 @@ opt_asc_desc: $$.val = tree.DefaultDirection } +alter_database_to_schema_stmt: + ALTER DATABASE database_name CONVERT TO SCHEMA WITH PARENT database_name + { + $$.val = &tree.ReparentDatabase{Name: tree.Name($3), Parent: tree.Name($9)} + } + alter_rename_database_stmt: ALTER DATABASE database_name RENAME TO database_name { @@ -11138,6 +11146,7 @@ unreserved_keyword: | CONFIGURE | CONSTRAINTS | CONVERSION +| CONVERT | COPY | COVERING | CREATEROLE diff --git a/pkg/sql/plan.go b/pkg/sql/plan.go index 38d2ba03f9ba..d67c195d9cd0 100644 --- a/pkg/sql/plan.go +++ b/pkg/sql/plan.go @@ -192,6 +192,7 @@ var _ planNode = &renameColumnNode{} var _ planNode = &renameDatabaseNode{} var _ planNode = &renameIndexNode{} var _ planNode = &renameTableNode{} +var _ planNode = &reparentDatabaseNode{} var _ planNode = &renderNode{} var _ planNode = &RevokeRoleNode{} var _ planNode = &rowCountNode{} @@ -237,6 +238,7 @@ var _ planNodeReadingOwnWrites = &changePrivilegesNode{} var _ planNodeReadingOwnWrites = &dropSchemaNode{} var _ planNodeReadingOwnWrites = &dropTypeNode{} var _ planNodeReadingOwnWrites = &refreshMaterializedViewNode{} +var _ planNodeReadingOwnWrites = &reparentDatabaseNode{} var _ planNodeReadingOwnWrites = &setZoneConfigNode{} // planNodeRequireSpool serves as marker for nodes whose parent must diff --git a/pkg/sql/reparent_database.go b/pkg/sql/reparent_database.go new file mode 100644 index 000000000000..30ad1066bfe9 --- /dev/null +++ b/pkg/sql/reparent_database.go @@ -0,0 +1,237 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package sql + +import ( + "context" + + "github.com/cockroachdb/cockroach/pkg/clusterversion" + "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkv" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/resolver" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/util/protoutil" + "github.com/cockroachdb/errors" +) + +type reparentDatabaseNode struct { + n *tree.ReparentDatabase + db *sqlbase.MutableDatabaseDescriptor + newParent *sqlbase.MutableDatabaseDescriptor +} + +func (p *planner) ReparentDatabase( + ctx context.Context, n *tree.ReparentDatabase, +) (planNode, error) { + // We'll only allow the admin to perform this reparenting action. + if err := p.RequireAdminRole(ctx, "ALTER DATABASE ... CONVERT TO SCHEMA"); err != nil { + return nil, err + } + + // Ensure that the cluster version is high enough to create the schema. + if !p.ExecCfg().Settings.Version.IsActive(ctx, clusterversion.VersionUserDefinedSchemas) { + return nil, pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, + `creating schemas requires all nodes to be upgraded to %s`, + clusterversion.VersionByKey(clusterversion.VersionUserDefinedSchemas)) + } + + // Check that creation of schemas is enabled. + if !p.EvalContext().SessionData.UserDefinedSchemasEnabled { + return nil, pgerror.Newf(pgcode.FeatureNotSupported, + "session variable experimental_enable_user_defined_schemas is set to false, cannot create a schema") + } + + // TODO (rohany): Need mutable DB access here. + db, err := p.ResolveUncachedDatabaseByName(ctx, string(n.Name), true /* required */) + if err != nil { + return nil, err + } + + parent, err := p.ResolveUncachedDatabaseByName(ctx, string(n.Parent), true /* required */) + if err != nil { + return nil, err + } + + // Ensure that this database wouldn't collide with a name under the new database. + exists, err := p.schemaExists(ctx, parent.ID, db.Name) + if err != nil { + return nil, err + } + if exists { + return nil, pgerror.Newf(pgcode.DuplicateSchema, "schema %q already exists", db.Name) + } + + // Ensure the database has a valid schema name. + if err := sqlbase.IsSchemaNameValid(db.Name); err != nil { + return nil, err + } + + // We can't reparent a database that has any child schemas other than public. + if len(db.Schemas) > 0 { + return nil, pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, "cannot convert database with schemas into schema") + } + + return &reparentDatabaseNode{ + n: n, + db: sqlbase.NewMutableExistingDatabaseDescriptor(*db.DatabaseDesc()), + newParent: sqlbase.NewMutableExistingDatabaseDescriptor(*parent.DatabaseDesc()), + }, nil +} + +func (n *reparentDatabaseNode) startExec(params runParams) error { + ctx, p, codec := params.ctx, params.p, params.ExecCfg().Codec + + // Make a new schema corresponding to the target db. + id, err := catalogkv.GenerateUniqueDescID(ctx, p.ExecCfg().DB, codec) + if err != nil { + return err + } + schema := sqlbase.NewMutableCreatedSchemaDescriptor(descpb.SchemaDescriptor{ + ParentID: n.newParent.ID, + Name: n.db.Name, + ID: id, + Privileges: protoutil.Clone(n.db.Privileges).(*descpb.PrivilegeDescriptor), + }) + // Add the new schema to the parent database's name map. + if n.newParent.Schemas == nil { + n.newParent.Schemas = make(map[string]descpb.DatabaseDescriptor_SchemaInfo) + } + n.newParent.Schemas[n.db.Name] = descpb.DatabaseDescriptor_SchemaInfo{ + ID: schema.ID, + Dropped: false, + } + + if err := p.createDescriptorWithID( + ctx, + sqlbase.NewSchemaKey(n.newParent.ID, schema.Name).Key(p.ExecCfg().Codec), + id, + schema, + params.ExecCfg().Settings, + tree.AsStringWithFQNames(n.n, params.Ann()), + ); err != nil { + return err + } + + b := p.txn.NewBatch() + + // Get all objects under the target database. + objNames, err := resolver.GetObjectNames(ctx, p.txn, p, codec, n.db, tree.PublicSchema, true /* explicitPrefix */) + if err != nil { + return err + } + + // For each object, adjust the ParentID and ParentSchemaID fields to point + // to the new parent DB and schema. + for _, objName := range objNames { + // First try looking up objName as a table. + found, desc, err := p.LookupObject( + ctx, + tree.ObjectLookupFlags{ + // Note we set required to be false here in order to not error out + // if we don't find the object. + CommonLookupFlags: tree.CommonLookupFlags{Required: false}, + RequireMutable: true, + IncludeOffline: true, + DesiredObjectKind: tree.TableObject, + }, + objName.Catalog(), + objName.Schema(), + objName.Object(), + ) + if err != nil { + return err + } + if found { + // Remap the ID's on the table. + tbl, ok := desc.(*sqlbase.MutableTableDescriptor) + if !ok { + return errors.AssertionFailedf("%q was not a MutableTableDescriptor", objName.Object()) + } + tbl.AddDrainingName(descpb.NameInfo{ + ParentID: tbl.ParentID, + ParentSchemaID: tbl.GetParentSchemaID(), + Name: tbl.Name, + }) + tbl.ParentID = n.newParent.ID + tbl.UnexposedParentSchemaID = schema.ID + objKey := catalogkv.MakeObjectNameKey(ctx, p.ExecCfg().Settings, tbl.ParentID, tbl.GetParentSchemaID(), tbl.Name).Key(codec) + b.CPut(objKey, tbl.ID, nil /* expected */) + if err := p.writeSchemaChange(ctx, tbl, descpb.InvalidMutationID, tree.AsStringWithFQNames(n.n, params.Ann())); err != nil { + return err + } + } else { + // If we couldn't resolve objName as a table, try a type. + found, desc, err := p.LookupObject( + ctx, + tree.ObjectLookupFlags{ + CommonLookupFlags: tree.CommonLookupFlags{Required: true}, + RequireMutable: true, + IncludeOffline: true, + DesiredObjectKind: tree.TypeObject, + }, + objName.Catalog(), + objName.Schema(), + objName.Object(), + ) + if err != nil { + return err + } + // If we couldn't find the object at all, then continue. + if !found { + continue + } + // Remap the ID's on the type. + typ, ok := desc.(*sqlbase.MutableTypeDescriptor) + if !ok { + return errors.AssertionFailedf("%q was not a MutableTypeDescriptor", objName.Object()) + } + typ.AddDrainingName(descpb.NameInfo{ + ParentID: typ.ParentID, + ParentSchemaID: typ.ParentSchemaID, + Name: typ.Name, + }) + typ.ParentID = n.newParent.ID + typ.ParentSchemaID = schema.ID + objKey := catalogkv.MakeObjectNameKey(ctx, p.ExecCfg().Settings, typ.ParentID, typ.ParentSchemaID, typ.Name).Key(codec) + b.CPut(objKey, typ.ID, nil /* expected */) + if err := p.writeTypeSchemaChange(ctx, typ, tree.AsStringWithFQNames(n.n, params.Ann())); err != nil { + return err + } + } + } + + // Delete the public schema namespace entry for this database. Per our check + // during initialization, this is the only schema present under n.db. + b.Del(catalogkv.MakeObjectNameKey(ctx, p.ExecCfg().Settings, n.db.ID, keys.RootNamespaceID, tree.PublicSchema).Key(codec)) + + // Delete the namespace entry for the database. + // TODO (rohany): I need the database cache changes to go in before I can drain + // the name here. + b.Del(catalogkv.MakeDatabaseNameKey(ctx, p.ExecCfg().Settings, n.db.Name).Key(codec)) + b.Del(sqlbase.MakeDescMetadataKey(codec, n.db.ID)) + p.Descriptors().AddUncommittedDatabase(n.db.Name, n.db.ID, descs.DBDropped) + + if err := p.txn.Run(ctx, b); err != nil { + return err + } + + return p.writeDatabaseChange(ctx, n.newParent) +} + +func (n *reparentDatabaseNode) Next(params runParams) (bool, error) { return false, nil } +func (n *reparentDatabaseNode) Values() tree.Datums { return tree.Datums{} } +func (n *reparentDatabaseNode) Close(ctx context.Context) {} +func (n *reparentDatabaseNode) ReadingOwnWrites() {} diff --git a/pkg/sql/sem/tree/rename.go b/pkg/sql/sem/tree/rename.go index ff857e1ad3f1..1f6766325b57 100644 --- a/pkg/sql/sem/tree/rename.go +++ b/pkg/sql/sem/tree/rename.go @@ -33,6 +33,20 @@ func (node *RenameDatabase) Format(ctx *FmtCtx) { ctx.FormatNode(&node.NewName) } +// ReparentDatabase represents a database reparenting as a schema operation. +type ReparentDatabase struct { + Name Name + Parent Name +} + +// Format implements the NodeFormatter interface. +func (node *ReparentDatabase) Format(ctx *FmtCtx) { + ctx.WriteString("ALTER DATABASE ") + node.Name.Format(ctx) + ctx.WriteString(" CONVERT TO SCHEMA WITH PARENT ") + node.Parent.Format(ctx) +} + // RenameTable represents a RENAME TABLE or RENAME VIEW or RENAME SEQUENCE // statement. Whether the user has asked to rename a view or a sequence // is indicated by the IsView and IsSequence fields. diff --git a/pkg/sql/sem/tree/stmt.go b/pkg/sql/sem/tree/stmt.go index ea44430d36f0..4bb69d4ad474 100644 --- a/pkg/sql/sem/tree/stmt.go +++ b/pkg/sql/sem/tree/stmt.go @@ -584,6 +584,12 @@ func (*RenameDatabase) StatementType() StatementType { return DDL } // StatementTag returns a short string identifying the type of statement. func (*RenameDatabase) StatementTag() string { return "RENAME DATABASE" } +// StatementType implements the Statement interface. +func (*ReparentDatabase) StatementType() StatementType { return DDL } + +// StatementTag returns a short string identifying the type of statement. +func (*ReparentDatabase) StatementTag() string { return "TODO (rohany): Implement" } + // StatementType implements the Statement interface. func (*RenameIndex) StatementType() StatementType { return DDL } @@ -1042,6 +1048,7 @@ func (n *Relocate) String() string { return AsString(n) } func (n *RefreshMaterializedView) String() string { return AsString(n) } func (n *RenameColumn) String() string { return AsString(n) } func (n *RenameDatabase) String() string { return AsString(n) } +func (n *ReparentDatabase) String() string { return AsString(n) } func (n *RenameIndex) String() string { return AsString(n) } func (n *RenameTable) String() string { return AsString(n) } func (n *Restore) String() string { return AsString(n) } diff --git a/pkg/sql/walk.go b/pkg/sql/walk.go index d42d3d3af83b..4ddad7a59c04 100644 --- a/pkg/sql/walk.go +++ b/pkg/sql/walk.go @@ -402,6 +402,7 @@ var planNodeNames = map[reflect.Type]string{ reflect.TypeOf(&renameDatabaseNode{}): "rename database", reflect.TypeOf(&renameIndexNode{}): "rename index", reflect.TypeOf(&renameTableNode{}): "rename table", + reflect.TypeOf(&reparentDatabaseNode{}): "TODO (rohany): fill out", reflect.TypeOf(&renderNode{}): "render", reflect.TypeOf(&RevokeRoleNode{}): "revoke role", reflect.TypeOf(&rowCountNode{}): "count",