From ca0faae830bb2e5e0e0b6d614dd21187d167ff46 Mon Sep 17 00:00:00 2001 From: arulajmani Date: Fri, 8 Oct 2021 20:05:19 -0400 Subject: [PATCH] spanconfig: introduce the spanconfig.SQLTranslator This patch introduces the spanconfig.SQLTranslator which translates a descriptor and its corresponding zone configurations into its constituent span configurations. The zone config and span config protos look similar, but they differ from each other in a couple of ways: - zone configurations correspond to {descriptor, zone} IDs whereas span configurations correspond to keyspans. - zone configurations have a notion of inheritance which span configurations do not. Instead, they're fully hydrated and flattened out. When the SQLTranslator is given a {zone,descriptor} ID it first descends the zone configuration hierarchy starting from the ID to accumulate IDs of all leaf objects. Leaf objects are tables and named zones (other than RANGE DEFAULT) which have actual span configurations associated with them. For every one of these accumulated IDs, it then generates tuples by following up the inheritance chain to fully hydrate the span config. The SQLTranslator also accounts for and negotiates subzones in this process. Concretely, for the following zone configuration hierarchy: ``` CREATE DATABASE db; CREATE TABLE db.t1(); ALTER DATABASE db CONFIGURE ZONE USING num_replicas=7; ALTER TABLE db.t1 CONFIGURE ZONE USING num_voters=5; ``` The SQLTranslator produces the following translation (represented as a diff against RANGE DEFAULT for brevity): ``` /Table/5{3-4} num_replicas=7 num_voters=5 ``` Split out from #69661. Release note: None --- .github/CODEOWNERS | 3 +- pkg/BUILD.bazel | 1 + .../changefeedccl/helpers_tenant_shim_test.go | 1 + pkg/ccl/spanconfigccl/BUILD.bazel | 36 ++ pkg/ccl/spanconfigccl/datadriven_test.go | 279 +++++++++++++ pkg/ccl/spanconfigccl/main_test.go | 33 ++ pkg/ccl/spanconfigccl/testdata/databases | 49 +++ pkg/ccl/spanconfigccl/testdata/full_translate | 66 +++ .../full_translate_named_zones_deleted | 79 ++++ pkg/ccl/spanconfigccl/testdata/indexes | 94 +++++ pkg/ccl/spanconfigccl/testdata/misc | 92 +++++ pkg/ccl/spanconfigccl/testdata/named_zones | 45 ++ .../testdata/partitions_primary_index | 138 +++++++ pkg/config/zonepb/BUILD.bazel | 1 + pkg/config/zonepb/zone.go | 87 ++-- pkg/config/zonepb/zone_test.go | 47 +-- pkg/keys/spans.go | 3 + pkg/roachpb/span_config.go | 2 + pkg/server/BUILD.bazel | 1 + pkg/server/server_sql.go | 3 + pkg/server/testserver.go | 10 + pkg/spanconfig/BUILD.bazel | 3 + pkg/spanconfig/spanconfig.go | 56 ++- pkg/spanconfig/spanconfigmanager/manager.go | 17 +- .../spanconfigmanager/manager_test.go | 3 + .../spanconfigsqltranslator/BUILD.bazel | 22 + .../spanconfigsqltranslator/sql_translator.go | 385 ++++++++++++++++++ pkg/sql/catalog/descs/collection.go | 34 ++ pkg/sql/show_zone_config.go | 2 +- pkg/sql/zone_config.go | 90 ++++ pkg/testutils/serverutils/test_server_shim.go | 4 + 31 files changed, 1617 insertions(+), 69 deletions(-) create mode 100644 pkg/ccl/spanconfigccl/BUILD.bazel create mode 100644 pkg/ccl/spanconfigccl/datadriven_test.go create mode 100644 pkg/ccl/spanconfigccl/main_test.go create mode 100644 pkg/ccl/spanconfigccl/testdata/databases create mode 100644 pkg/ccl/spanconfigccl/testdata/full_translate create mode 100644 pkg/ccl/spanconfigccl/testdata/full_translate_named_zones_deleted create mode 100644 pkg/ccl/spanconfigccl/testdata/indexes create mode 100644 pkg/ccl/spanconfigccl/testdata/misc create mode 100644 pkg/ccl/spanconfigccl/testdata/named_zones create mode 100644 pkg/ccl/spanconfigccl/testdata/partitions_primary_index create mode 100644 pkg/spanconfig/spanconfigsqltranslator/BUILD.bazel create mode 100644 pkg/spanconfig/spanconfigsqltranslator/sql_translator.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ff597b8dcf08..94270c968b2b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -93,6 +93,7 @@ /pkg/ccl/backupccl/ @cockroachdb/bulk-prs /pkg/ccl/importccl/ @cockroachdb/bulk-prs +/pkg/ccl/spanconfigccl/ @cockroachdb/kv-prs /pkg/ccl/storageccl/ @cockroachdb/bulk-prs /pkg/cloud/ @cockroachdb/bulk-prs /pkg/sql/distsql_plan_csv.go @cockroachdb/bulk-prs @@ -221,7 +222,7 @@ /pkg/scheduledjobs/ @cockroachdb/bulk-prs /pkg/security/ @cockroachdb/server-prs @cockroachdb/prodsec /pkg/settings/ @cockroachdb/server-prs -/pkg/spanconfig/ @cockroachdb/multiregion +/pkg/spanconfig/ @cockroachdb/kv-prs /pkg/startupmigrations/ @cockroachdb/server-prs @cockroachdb/sql-schema /pkg/streaming/ @cockroachdb/bulk-prs /pkg/testutils/ @cockroachdb/test-eng-noreview diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index cb5fb2e97ce8..57df0b6fe386 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -32,6 +32,7 @@ ALL_TESTS = [ "//pkg/ccl/partitionccl:partitionccl_test", "//pkg/ccl/serverccl/diagnosticsccl:diagnosticsccl_test", "//pkg/ccl/serverccl:serverccl_test", + "//pkg/ccl/spanconfigccl:spanconfigccl_test", "//pkg/ccl/sqlproxyccl/denylist:denylist_test", "//pkg/ccl/sqlproxyccl/idle:idle_test", "//pkg/ccl/sqlproxyccl/tenant:tenant_test", diff --git a/pkg/ccl/changefeedccl/helpers_tenant_shim_test.go b/pkg/ccl/changefeedccl/helpers_tenant_shim_test.go index dd03cc0de33c..5f8617371a26 100644 --- a/pkg/ccl/changefeedccl/helpers_tenant_shim_test.go +++ b/pkg/ccl/changefeedccl/helpers_tenant_shim_test.go @@ -70,6 +70,7 @@ func (t *testServerShim) Clock() *hlc.Clock { panic(unsuppor func (t *testServerShim) DistSenderI() interface{} { panic(unsupportedShimMethod) } func (t *testServerShim) MigrationServer() interface{} { panic(unsupportedShimMethod) } func (t *testServerShim) SpanConfigAccessor() interface{} { panic(unsupportedShimMethod) } +func (t *testServerShim) SpanConfigSQLTranslator() interface{} { panic(unsupportedShimMethod) } func (t *testServerShim) SQLServer() interface{} { panic(unsupportedShimMethod) } func (t *testServerShim) SQLLivenessProvider() interface{} { panic(unsupportedShimMethod) } func (t *testServerShim) StartupMigrationsManager() interface{} { panic(unsupportedShimMethod) } diff --git a/pkg/ccl/spanconfigccl/BUILD.bazel b/pkg/ccl/spanconfigccl/BUILD.bazel new file mode 100644 index 000000000000..60b773d08cbf --- /dev/null +++ b/pkg/ccl/spanconfigccl/BUILD.bazel @@ -0,0 +1,36 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "spanconfigccl_test", + srcs = [ + "datadriven_test.go", + "main_test.go", + ], + data = glob(["testdata/**"]), + deps = [ + "//pkg/base", + "//pkg/ccl/partitionccl", + "//pkg/ccl/utilccl", + "//pkg/config/zonepb", + "//pkg/kv", + "//pkg/roachpb:with-mocks", + "//pkg/security", + "//pkg/security/securitytest", + "//pkg/server", + "//pkg/spanconfig", + "//pkg/sql", + "//pkg/sql/catalog/catalogkv", + "//pkg/sql/catalog/descpb", + "//pkg/sql/catalog/descs", + "//pkg/sql/catalog/tabledesc", + "//pkg/sql/sem/tree", + "//pkg/testutils/serverutils", + "//pkg/testutils/sqlutils", + "//pkg/testutils/testcluster", + "//pkg/util/leaktest", + "//pkg/util/log", + "//pkg/util/randutil", + "@com_github_cockroachdb_datadriven//:datadriven", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/ccl/spanconfigccl/datadriven_test.go b/pkg/ccl/spanconfigccl/datadriven_test.go new file mode 100644 index 000000000000..ad3f29dc95e8 --- /dev/null +++ b/pkg/ccl/spanconfigccl/datadriven_test.go @@ -0,0 +1,279 @@ +// Copyright 2021 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package spanconfigccl_test + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/cockroachdb/cockroach/pkg/base" + _ "github.com/cockroachdb/cockroach/pkg/ccl/partitionccl" + "github.com/cockroachdb/cockroach/pkg/config/zonepb" + "github.com/cockroachdb/cockroach/pkg/kv" + "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/sql" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkv" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" + "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/datadriven" + "github.com/stretchr/testify/require" +) + +// TestSQLTranslatorDataDriven is a data-driven test for the +// spanconfigsqltranslator.SQLTranslator. It allows users to set up zone config +// hierarchies and validate their translation to SpanConfigs is as expected. +// Only fields that are different from the (default) RANGE DEFAULT are printed +// in the test output for readability. +// +// It offers the following commands: +// +// "exec-sql": executes the input SQL query. +// +// "query-sql": executes the input SQL query and prints the results. +// +// "translate [database=] [table=] [named-zone=] +// [id=]: +// translates the SQL zone config state to the span config state starting from +// the referenced object (named zone, database, database + table, or descriptor +// id) as the root. +// +// "full-translate": performs a full translation of the SQL zone config state +// to the implied span config state. +// +// "sleep" [duration=]: sleep for the provided duration. +// +// "mark-table-offline" [database=] [table=]: marks the given +// table as offline for testing purposes. +// +// "mark-table-public" [database=] [table=]: marks the given +// table as public. +// +// TODO(arul): Add a secondary tenant configuration for this test as well. +func TestSQLTranslatorDataDriven(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + ctx := context.Background() + datadriven.Walk(t, "testdata/", func(t *testing.T, path string) { + tc := testcluster.StartTestCluster(t, 1, base.TestClusterArgs{ + ServerArgs: base.TestServerArgs{ + EnableSpanConfigs: true, + Knobs: base.TestingKnobs{ + SpanConfig: &spanconfig.TestingKnobs{ + ManagerDisableJobCreation: true, + }, + }, + }, + }) + defer tc.Stopper().Stop(ctx) + sqlDB := tc.ServerConn(0 /* idx */) + + sqlTranslator := tc.Server(0).SpanConfigSQLTranslator().(spanconfig.SQLTranslator) + datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { + switch d.Cmd { + case "exec-sql": + _, err := sqlDB.Exec(d.Input) + if err != nil { + return err.Error() + } + case "query-sql": + rows, err := sqlDB.Query(d.Input) + if err != nil { + return err.Error() + } + output, err := sqlutils.RowsToDataDrivenOutput(rows) + require.NoError(t, err) + return output + case "translate": + // Parse the args to get the object ID we're interested in translating. + objID := descpb.InvalidID + if d.HasArg(namedZone) { + var zone string + d.ScanArgs(t, namedZone, &zone) + namedZoneID, found := zonepb.NamedZones[zonepb.NamedZone(zone)] + require.Truef(t, found, "unknown named zone: %s", zone) + objID = descpb.ID(namedZoneID) + } else if d.HasArg(database) { + var dbName string + d.ScanArgs(t, database, &dbName) + if d.HasArg(table) { + var tbName string + d.ScanArgs(t, table, &tbName) + tableDesc := catalogkv.TestingGetTableDescriptor( + tc.Server(0).DB(), + tc.Server(0).ExecutorConfig().(sql.ExecutorConfig).Codec, + dbName, + tbName, + ) + objID = tableDesc.GetID() + } else { + dbDesc := catalogkv.TestingGetDatabaseDescriptor( + tc.Server(0).DB(), + tc.Server(0).ExecutorConfig().(sql.ExecutorConfig).Codec, + dbName, + ) + objID = dbDesc.GetID() + } + } else if d.HasArg(id) { + var scanID int + d.ScanArgs(t, id, &scanID) + objID = descpb.ID(scanID) + } else { + t.Fatal("insufficient args provided to translate") + } + entries, _, err := sqlTranslator.Translate(ctx, descpb.IDs{objID}) + require.NoError(t, err) + return datadrivenTranslationResult(entries) + case "full-translate": + entries, _, err := spanconfig.FullTranslate(ctx, sqlTranslator) + require.NoError(t, err) + return datadrivenTranslationResult(entries) + case "sleep": + var sleepDuration int + d.ScanArgs(t, duration, &sleepDuration) + time.Sleep(time.Second * time.Duration(sleepDuration)) + case "mark-table-offline": + var dbName string + d.ScanArgs(t, database, &dbName) + var tbName string + d.ScanArgs(t, table, &tbName) + err := modifyTableDescriptor(ctx, tc, dbName, tbName, func(mutable *tabledesc.Mutable) { + mutable.SetOffline("for testing") + }) + require.NoError(t, err) + case "mark-table-public": + var dbName string + d.ScanArgs(t, database, &dbName) + var tbName string + d.ScanArgs(t, table, &tbName) + err := modifyTableDescriptor(ctx, tc, dbName, tbName, func(mutable *tabledesc.Mutable) { + mutable.SetPublic() + }) + require.NoError(t, err) + default: + t.Fatalf("unknown command: %s", d.Cmd) + } + return "" + }) + }) +} + +// Constants for data-driven args. +const ( + id = "id" + namedZone = "named-zone" + table = "table" + database = "database" + duration = "duration" +) + +func modifyTableDescriptor( + ctx context.Context, + tc *testcluster.TestCluster, + dbName string, + tbName string, + f func(*tabledesc.Mutable), +) error { + cfg := tc.Server(0).ExecutorConfig().(sql.ExecutorConfig) + return sql.DescsTxn(ctx, &cfg, func( + ctx context.Context, txn *kv.Txn, descsCol *descs.Collection, + ) error { + _, tableDesc, err := descsCol.GetMutableTableByName( + ctx, + txn, + tree.NewTableNameWithSchema(tree.Name(dbName), "public", tree.Name(tbName)), + tree.ObjectLookupFlags{ + CommonLookupFlags: tree.CommonLookupFlags{ + Required: true, + IncludeOffline: true, + }, + }, + ) + if err != nil { + return err + } + f(tableDesc) + return descsCol.WriteDesc(ctx, false, tableDesc, txn) + }) +} + +// datadrivenTranslationResult constructs the datadriven output for a given +// slice of (translated) span config entries. +func datadrivenTranslationResult(entries []roachpb.SpanConfigEntry) string { + var output strings.Builder + for _, entry := range entries { + res := diffEntryAgainstRangeDefault(entry) + output.WriteString(res) + output.WriteByte('\n') + } + return output.String() +} + +// diffEntryAgainstRangeDefault computes the difference between the given config +// and RANGE DEFAULT. It then constructs a (span<->mismatching field(s)) string +// and returns it. If there config is same as RANGE DEFAULT, a (span, DEFAULT) +// string is returned instead. +func diffEntryAgainstRangeDefault(entry roachpb.SpanConfigEntry) string { + var res strings.Builder + defaultSpanConfig := roachpb.TestingDefaultSpanConfig() + var diffs []string + + if entry.Config.RangeMaxBytes != defaultSpanConfig.RangeMaxBytes { + diffs = append(diffs, fmt.Sprintf("range_max_bytes=%d", entry.Config.RangeMaxBytes)) + } + if entry.Config.RangeMinBytes != defaultSpanConfig.RangeMinBytes { + diffs = append(diffs, fmt.Sprintf("range_min_bytes=%d", entry.Config.RangeMinBytes)) + } + if entry.Config.GCPolicy.TTLSeconds != defaultSpanConfig.GCPolicy.TTLSeconds { + diffs = append(diffs, fmt.Sprintf("ttl_seconds=%d", entry.Config.GCPolicy.TTLSeconds)) + } + if entry.Config.GlobalReads != defaultSpanConfig.GlobalReads { + diffs = append(diffs, fmt.Sprintf("global_reads=%v", entry.Config.GlobalReads)) + } + if entry.Config.NumReplicas != defaultSpanConfig.NumReplicas { + diffs = append(diffs, fmt.Sprintf("num_replicas=%d", entry.Config.NumReplicas)) + } + if entry.Config.NumVoters != defaultSpanConfig.NumVoters { + diffs = append(diffs, fmt.Sprintf("num_voters=%d", entry.Config.NumVoters)) + } + if !reflect.DeepEqual(entry.Config.Constraints, defaultSpanConfig.Constraints) { + diffs = append(diffs, fmt.Sprintf("constraints=%v", entry.Config.Constraints)) + } + if !reflect.DeepEqual(entry.Config.VoterConstraints, defaultSpanConfig.VoterConstraints) { + diffs = append(diffs, fmt.Sprintf("voter_constraints=%v", entry.Config.VoterConstraints)) + } + if !reflect.DeepEqual(entry.Config.LeasePreferences, defaultSpanConfig.LeasePreferences) { + diffs = append(diffs, fmt.Sprintf("lease_preferences=%v", entry.Config.VoterConstraints)) + } + + if len(diffs) != 0 { + for i, diff := range diffs { + if i == 0 { + res.WriteString(fmt.Sprintf("%-30s %s", entry.Span.String(), diff)) + } else { + res.WriteByte(' ') + res.WriteString(diff) + } + } + } else { + res.WriteString(fmt.Sprintf("%-30s %s", entry.Span.String(), "DEFAULT")) + } + return res.String() +} diff --git a/pkg/ccl/spanconfigccl/main_test.go b/pkg/ccl/spanconfigccl/main_test.go new file mode 100644 index 000000000000..ac929a3c3a94 --- /dev/null +++ b/pkg/ccl/spanconfigccl/main_test.go @@ -0,0 +1,33 @@ +// Copyright 2021 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package spanconfigccl_test + +import ( + "os" + "testing" + + "github.com/cockroachdb/cockroach/pkg/ccl/utilccl" + "github.com/cockroachdb/cockroach/pkg/security" + "github.com/cockroachdb/cockroach/pkg/security/securitytest" + "github.com/cockroachdb/cockroach/pkg/server" + "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" + "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" + "github.com/cockroachdb/cockroach/pkg/util/randutil" +) + +//go:generate ../../util/leaktest/add-leaktest.sh *_test.go + +func TestMain(m *testing.M) { + defer utilccl.TestingEnableEnterprise()() + security.SetAssetLoader(securitytest.EmbeddedAssets) + randutil.SeedForTests() + serverutils.InitTestServerFactory(server.TestServerFactory) + serverutils.InitTestClusterFactory(testcluster.TestClusterFactory) + os.Exit(m.Run()) +} diff --git a/pkg/ccl/spanconfigccl/testdata/databases b/pkg/ccl/spanconfigccl/testdata/databases new file mode 100644 index 000000000000..4f4410e13de3 --- /dev/null +++ b/pkg/ccl/spanconfigccl/testdata/databases @@ -0,0 +1,49 @@ +# Create a database with some tables, types, and schemas. Check that span +# configurations are as we expect. + +exec-sql +CREATE DATABASE db; +CREATE TABLE db.t1(); +CREATE TYPE db.typ AS ENUM(); +CREATE SCHEMA db.sc; +CREATE TABLE db.t2(); +---- + +query-sql +SELECT id FROM system.namespace WHERE name='t1' +---- +53 + +query-sql +SELECT id FROM system.namespace WHERE name='t2' +---- +57 + +# We only expect there to be span config entries for tables t1 and t2. +translate database=db +---- +/Table/5{3-4} DEFAULT +/Table/5{7-8} DEFAULT + +# Alter zone config fields on the database and one of the tables to ensure +# things are cascading. +exec-sql +ALTER DATABASE db CONFIGURE ZONE USING num_replicas=7; +ALTER TABLE db.t1 CONFIGURE ZONE USING num_voters=5; +---- + +translate database=db +---- +/Table/5{3-4} num_replicas=7 num_voters=5 +/Table/5{7-8} num_replicas=7 + +# Translating the tables in the database individually should result in the same +# config as above. + +translate database=db table=t1 +---- +/Table/5{3-4} num_replicas=7 num_voters=5 + +translate database=db table=t2 +---- +/Table/5{7-8} num_replicas=7 diff --git a/pkg/ccl/spanconfigccl/testdata/full_translate b/pkg/ccl/spanconfigccl/testdata/full_translate new file mode 100644 index 000000000000..624d8ee21522 --- /dev/null +++ b/pkg/ccl/spanconfigccl/testdata/full_translate @@ -0,0 +1,66 @@ +# This file tests a full reconciliation scenario when none of the named zone +# entries have been messed with and the user has created a single database +# and a single table inside that database. + +exec-sql +CREATE DATABASE db; +CREATE SCHEMA sc; +CREATE TYPE typ AS ENUM(); +CREATE TABLE db.t(); +---- + +# - User created table above, with ID 56. +# There should be no entry for IDs 52, 53, 54, and 55 as these belong to the +# database, schema, type, and type alias respectively. +# - All system tables. Note that there should be no entry for pseudo IDs or IDs +# for which no table exists. +# - NodeLivenessSpan +# - Meta ranges: min -> NodeLiveness start +# - System ranges: +# - NodeLiveness end -> TimeSeries Start +# - TimeSeries end -> System Ranges end +# - Time Series Span +full-translate +---- +/Table/5{6-7} DEFAULT +/Table/{3-4} num_replicas=5 +/Table/{4-5} num_replicas=5 +/Table/{5-6} num_replicas=5 +/Table/{6-7} num_replicas=5 +/Table/{8-9} num_replicas=5 +/Table/1{1-2} num_replicas=5 +/Table/1{2-3} num_replicas=5 +/Table/1{3-4} num_replicas=5 +/Table/1{4-5} num_replicas=5 +/Table/1{5-6} num_replicas=5 +/Table/{19-20} num_replicas=5 +/Table/2{0-1} num_replicas=5 +/Table/2{1-2} num_replicas=5 +/Table/2{3-4} num_replicas=5 +/Table/2{4-5} num_replicas=5 +/Table/2{5-6} ttl_seconds=600 num_replicas=5 +/Table/2{6-7} num_replicas=5 +/Table/2{7-8} ttl_seconds=600 num_replicas=5 +/Table/2{8-9} num_replicas=5 +/NamespaceTable/{30-Max} num_replicas=5 +/{NamespaceTable/Max-Table/32} num_replicas=5 +/Table/3{2-3} num_replicas=5 +/Table/3{3-4} num_replicas=5 +/Table/3{4-5} num_replicas=5 +/Table/3{5-6} num_replicas=5 +/Table/3{6-7} num_replicas=5 +/Table/3{7-8} num_replicas=5 +/Table/{39-40} num_replicas=5 +/Table/4{0-1} num_replicas=5 +/Table/4{1-2} num_replicas=5 +/Table/4{2-3} num_replicas=5 +/Table/4{3-4} num_replicas=5 +/Table/4{4-5} num_replicas=5 +/Table/4{5-6} ttl_seconds=7200 num_replicas=5 +/Table/4{6-7} num_replicas=5 +/Table/4{7-8} num_replicas=5 +/System/NodeLiveness{-Max} ttl_seconds=600 num_replicas=5 +/{Min-System/NodeLiveness} ttl_seconds=3600 num_replicas=5 +/System/{NodeLivenessMax-tsd} num_replicas=5 +/System{tse-/Max} num_replicas=5 +/System{/tsd-tse} DEFAULT diff --git a/pkg/ccl/spanconfigccl/testdata/full_translate_named_zones_deleted b/pkg/ccl/spanconfigccl/testdata/full_translate_named_zones_deleted new file mode 100644 index 000000000000..5095938c5860 --- /dev/null +++ b/pkg/ccl/spanconfigccl/testdata/full_translate_named_zones_deleted @@ -0,0 +1,79 @@ +# This file tests a full translation when all named zone entries have been +# removed from system.zones. The expectation is for us to still generate span +# config updates for these named zones. These should be the same as RANGE +# DEFAULT because that's what they inherit from in the absence of an explicit +# zone config. + +query-sql +SELECT id FROM system.zones +---- +0 +1 +16 +17 +22 +25 +27 +45 + +# Note that discarding RANGE_DEFAULT isn't allowed. +exec-sql +ALTER RANGE liveness CONFIGURE ZONE DISCARD; +ALTER RANGE meta CONFIGURE ZONE DISCARD; +ALTER RANGE system CONFIGURE ZONE DISCARD; +ALTER RANGE timeseries CONFIGURE ZONE DISCARD; +---- + +query-sql +SELECT id FROM system.zones +---- +0 +1 +25 +27 +45 + + +full-translate +---- +/Table/{3-4} num_replicas=5 +/Table/{4-5} num_replicas=5 +/Table/{5-6} num_replicas=5 +/Table/{6-7} num_replicas=5 +/Table/{8-9} num_replicas=5 +/Table/1{1-2} num_replicas=5 +/Table/1{2-3} num_replicas=5 +/Table/1{3-4} num_replicas=5 +/Table/1{4-5} num_replicas=5 +/Table/1{5-6} num_replicas=5 +/Table/{19-20} num_replicas=5 +/Table/2{0-1} num_replicas=5 +/Table/2{1-2} num_replicas=5 +/Table/2{3-4} num_replicas=5 +/Table/2{4-5} num_replicas=5 +/Table/2{5-6} ttl_seconds=600 num_replicas=5 +/Table/2{6-7} num_replicas=5 +/Table/2{7-8} ttl_seconds=600 num_replicas=5 +/Table/2{8-9} num_replicas=5 +/NamespaceTable/{30-Max} num_replicas=5 +/{NamespaceTable/Max-Table/32} num_replicas=5 +/Table/3{2-3} num_replicas=5 +/Table/3{3-4} num_replicas=5 +/Table/3{4-5} num_replicas=5 +/Table/3{5-6} num_replicas=5 +/Table/3{6-7} num_replicas=5 +/Table/3{7-8} num_replicas=5 +/Table/{39-40} num_replicas=5 +/Table/4{0-1} num_replicas=5 +/Table/4{1-2} num_replicas=5 +/Table/4{2-3} num_replicas=5 +/Table/4{3-4} num_replicas=5 +/Table/4{4-5} num_replicas=5 +/Table/4{5-6} ttl_seconds=7200 num_replicas=5 +/Table/4{6-7} num_replicas=5 +/Table/4{7-8} num_replicas=5 +/System/NodeLiveness{-Max} DEFAULT +/{Min-System/NodeLiveness} DEFAULT +/System/{NodeLivenessMax-tsd} DEFAULT +/System{tse-/Max} DEFAULT +/System{/tsd-tse} DEFAULT diff --git a/pkg/ccl/spanconfigccl/testdata/indexes b/pkg/ccl/spanconfigccl/testdata/indexes new file mode 100644 index 000000000000..a8d19aea1b7e --- /dev/null +++ b/pkg/ccl/spanconfigccl/testdata/indexes @@ -0,0 +1,94 @@ +# We start off by creating a simple database -> table -> index hierarchy. We set +# a zone configuration on the index but not on the table. This means the table +# has a "placeholder zone config". + +exec-sql +CREATE DATABASE db; +CREATE TABLE db.t(i INT PRIMARY KEY, j INT); +CREATE INDEX idx ON db.t (j); +ALTER DATABASE db CONFIGURE ZONE USING num_replicas=7; +ALTER INDEX db.t@idx CONFIGURE ZONE USING num_voters = 5; +---- + +query-sql +SHOW ZONE CONFIGURATION FOR DATABASE db +---- +DATABASE db ALTER DATABASE db CONFIGURE ZONE USING + range_min_bytes = 134217728, + range_max_bytes = 536870912, + gc.ttlseconds = 90000, + num_replicas = 7, + constraints = '[]', + lease_preferences = '[]' + +query-sql +SHOW ZONE CONFIGURATION FOR TABLE db.t +---- +DATABASE db ALTER DATABASE db CONFIGURE ZONE USING + range_min_bytes = 134217728, + range_max_bytes = 536870912, + gc.ttlseconds = 90000, + num_replicas = 7, + constraints = '[]', + lease_preferences = '[]' + +query-sql +SHOW ZONE CONFIGURATION FOR INDEX db.t@idx +---- +INDEX db.public.t@idx ALTER INDEX db.public.t@idx CONFIGURE ZONE USING + range_min_bytes = 134217728, + range_max_bytes = 536870912, + gc.ttlseconds = 90000, + num_replicas = 7, + num_voters = 5, + constraints = '[]', + voter_constraints = '[]', + lease_preferences = '[]' + +# First entry = primary index (table's config above) +# Second entry = index idx, so numvoters should be overridden. +# Third entry = Any future indexes that may be added to this table. Should be +# the same as the table's config. +translate database=db table=t +---- +/Table/53{-/2} num_replicas=7 +/Table/53/{2-3} num_replicas=7 num_voters=5 +/Table/5{3/3-4} num_replicas=7 + +# Configure GC ttl on the database and override it for the index. The table +# continues to hold a placeholder zone config. +exec-sql +ALTER DATABASE db CONFIGURE ZONE USING gc.ttlseconds = 3600; +ALTER INDEX db.t@idx CONFIGURE ZONE USING gc.ttlseconds = 25 +---- + +translate database=db table=t +---- +/Table/53{-/2} ttl_seconds=3600 num_replicas=7 +/Table/53/{2-3} ttl_seconds=25 num_replicas=7 num_voters=5 +/Table/5{3/3-4} ttl_seconds=3600 num_replicas=7 + +# Configure a zone config field on the table, so that it is no longer a +# placeholder zone config. +exec-sql +ALTER TABLE db.t CONFIGURE ZONE USING range_min_bytes = 1000, range_max_bytes=100000; +---- + +query-sql +SHOW ZONE CONFIGURATION FOR INDEX db.t@idx +---- +INDEX db.public.t@idx ALTER INDEX db.public.t@idx CONFIGURE ZONE USING + range_min_bytes = 1000, + range_max_bytes = 100000, + gc.ttlseconds = 25, + num_replicas = 7, + num_voters = 5, + constraints = '[]', + voter_constraints = '[]', + lease_preferences = '[]' + +translate database=db table=t +---- +/Table/53{-/2} range_max_bytes=100000 range_min_bytes=1000 ttl_seconds=3600 num_replicas=7 +/Table/53/{2-3} range_max_bytes=100000 range_min_bytes=1000 ttl_seconds=25 num_replicas=7 num_voters=5 +/Table/5{3/3-4} range_max_bytes=100000 range_min_bytes=1000 ttl_seconds=3600 num_replicas=7 diff --git a/pkg/ccl/spanconfigccl/testdata/misc b/pkg/ccl/spanconfigccl/testdata/misc new file mode 100644 index 000000000000..f44c68206530 --- /dev/null +++ b/pkg/ccl/spanconfigccl/testdata/misc @@ -0,0 +1,92 @@ +# Miscellaneous edge case tests for the SQLTranslator. + +# Test dropped tables/databases work correctly. + +exec-sql +CREATE DATABASE db; +CREATE TABLE db.t1(); +CREATE TABLE db.t2(); +ALTER TABLE db.t1 CONFIGURE ZONE USING gc.ttlseconds=1; +---- + +translate database=db +---- +/Table/5{3-4} ttl_seconds=1 +/Table/5{4-5} DEFAULT + +# Drop the table. +exec-sql +DROP TABLE db.t1; +---- + +# We still should be able to generate the span configuration for it when +# starting our translation from the database. +translate database=db +---- +/Table/5{3-4} ttl_seconds=1 +/Table/5{4-5} DEFAULT + +# Same as above, except this time the translation starts from the table's ID. +translate id=53 +---- +/Table/5{3-4} ttl_seconds=1 + +# Sleep for 5 seconds, which is more than the TTL on db.t1, so that the gc job +# can delete the descriptor. +sleep duration=5 +---- + +# By now t1's descriptor should have been deleted. +translate database=db +---- +/Table/5{4-5} DEFAULT + +# This no longer exists, so no span configuration should be generated. +translate id=53 +---- + +# Mark table t2 as offline, we should still be able to generate a span +# configuration for it. +mark-table-offline database=db table=t2 +---- + +# Should work for both when we start from the table and when we start from the +# table. +translate database=db table=t2 +---- +/Table/5{4-5} DEFAULT + +translate database=db +---- +/Table/5{4-5} DEFAULT + + +# Mark the table as public again. +mark-table-public database=db table=t2 +---- + +translate database=db table=t2 +---- +/Table/5{4-5} DEFAULT + +# Test schemas/types don't generate a span configuration. +exec-sql +CREATE SCHEMA db.sc; +CREATE TYPE db.typ AS ENUM(); +---- + +# Schema. +translate id=55 +---- + +# Enum. +translate id=56 +---- + +# Array type alias. +translate id=57 +---- + +# Test that non-existent IDs do not generate span configurations either. +translate id=500 +---- diff --git a/pkg/ccl/spanconfigccl/testdata/named_zones b/pkg/ccl/spanconfigccl/testdata/named_zones new file mode 100644 index 000000000000..9ef8220e53bc --- /dev/null +++ b/pkg/ccl/spanconfigccl/testdata/named_zones @@ -0,0 +1,45 @@ +# This test generates span configurations for various named zone IDs. No tests +# for RANGE DEFAULT are included here as generating span configurations for +# RANGE DEFAULT is akin to a full reconciliation. + + +# ID 18 is for the time series range which is the only named zone that doesn't +# have an entry in `system.zones` at bootstrap. It should inherit from RANGE +# DEFAULT. +query-sql +SELECT count(*) FROM system.zones WHERE id=18 +---- +0 + +translate named-zone=timeseries +---- +/System{/tsd-tse} DEFAULT + +# Adding an explicit zone configuration for the timeseries range should work +# as expected. +exec-sql +ALTER RANGE timeseries CONFIGURE ZONE USING gc.ttlseconds=1000 +---- + +translate named-zone=timeseries +---- +/System{/tsd-tse} ttl_seconds=1000 + +# Change a field on the liveness range and ensure it behaves as expected. +exec-sql +ALTER RANGE liveness CONFIGURE ZONE USING num_replicas=7; +---- + +translate named-zone=liveness +---- +/System/NodeLiveness{-Max} ttl_seconds=600 num_replicas=7 + +# We are allowed to discard the liveness range's zone configuration. The +# generated span should have the RANGE DEFAULT config. +exec-sql +ALTER RANGE liveness CONFIGURE ZONE DISCARD +---- + +translate named-zone=liveness +---- +/System/NodeLiveness{-Max} DEFAULT diff --git a/pkg/ccl/spanconfigccl/testdata/partitions_primary_index b/pkg/ccl/spanconfigccl/testdata/partitions_primary_index new file mode 100644 index 000000000000..69e0c62377fc --- /dev/null +++ b/pkg/ccl/spanconfigccl/testdata/partitions_primary_index @@ -0,0 +1,138 @@ +# This test creates partitions on a table's primary index and moves the zone +# configurations on these partitions through the steps described inline, making +# assertions along the way. + +exec-sql +CREATE DATABASE db; +CREATE TABLE db.t(i INT PRIMARY KEY, j INT) PARTITION BY LIST (i) ( + PARTITION one_two VALUES IN (1, 2), + PARTITION three_four VALUES IN (3, 4), + PARTITION default VALUES IN (default) +); +ALTER DATABASE db CONFIGURE ZONE USING num_replicas=7; +ALTER TABLE db.t CONFIGURE ZONE USING num_voters=5; +---- + +query-sql +SHOW ZONE CONFIGURATION FOR DATABASE db +---- +DATABASE db ALTER DATABASE db CONFIGURE ZONE USING + range_min_bytes = 134217728, + range_max_bytes = 536870912, + gc.ttlseconds = 90000, + num_replicas = 7, + constraints = '[]', + lease_preferences = '[]' + +query-sql +SHOW ZONE CONFIGURATION FOR TABLE db.t +---- +TABLE db.public.t ALTER TABLE db.public.t CONFIGURE ZONE USING + range_min_bytes = 134217728, + range_max_bytes = 536870912, + gc.ttlseconds = 90000, + num_replicas = 7, + num_voters = 5, + constraints = '[]', + voter_constraints = '[]', + lease_preferences = '[]' + +query-sql +SHOW ZONE CONFIGURATION FOR PARTITION one_two OF TABLE db.t +---- +TABLE db.public.t ALTER TABLE db.public.t CONFIGURE ZONE USING + range_min_bytes = 134217728, + range_max_bytes = 536870912, + gc.ttlseconds = 90000, + num_replicas = 7, + num_voters = 5, + constraints = '[]', + voter_constraints = '[]', + lease_preferences = '[]' + + +# There is no zone configuration set on any of the partitions initially. So +# there is just one entry here which covers the entire table's span. +translate database=db table=t +---- +/Table/5{3-4} num_replicas=7 num_voters=5 + + +exec-sql +ALTER PARTITION one_two OF TABLE db.t CONFIGURE ZONE USING global_reads=true +---- + +# Now that we have a zone configuration on one of the partitions, `one_two`, +# which implies two (adjacent) spans. Both these configs have global reads set +# to true. The table's spans before and after the partitions span continue to +# have the table's zone configuration. +translate database=db table=t +---- +/Table/53{-/1/1} num_replicas=7 num_voters=5 +/Table/53/1/{1-2} global_reads=true num_replicas=7 num_voters=5 +/Table/53/1/{2-3} global_reads=true num_replicas=7 num_voters=5 +/Table/5{3/1/3-4} num_replicas=7 num_voters=5 + +# Change two fields on the second partition. One of them (num_voters) is an +# override on the value set on the database's zone config. The other, gc.ttlseconds, +# is not set on either the table or the database. +exec-sql +ALTER PARTITION three_four OF TABLE db.t CONFIGURE ZONE USING gc.ttlseconds=5; +ALTER PARTITION three_four OF TABLE db.t CONFIGURE ZONE USING num_voters=3 +---- + +# We expect 2 more (adjacent) spans for the second partition now. These should +# have the correct values of gc.ttlseconds (5) and num_voters (3). +translate database=db table=t +---- +/Table/53{-/1/1} num_replicas=7 num_voters=5 +/Table/53/1/{1-2} global_reads=true num_replicas=7 num_voters=5 +/Table/53/1/{2-3} global_reads=true num_replicas=7 num_voters=5 +/Table/53/1/{3-4} ttl_seconds=5 num_replicas=7 num_voters=3 +/Table/53/1/{4-5} ttl_seconds=5 num_replicas=7 num_voters=3 +/Table/5{3/1/5-4} num_replicas=7 num_voters=5 + +exec-sql +ALTER PARTITION default OF TABLE db.t CONFIGURE ZONE USING num_voters=6 +---- + +# Now that we've set a zone configuration on the default partition we get a few +# more entries. Of note is that this default partition is on the primary index. +# This results in a span before the primary index starts +# (which will always be empty) but has the span config of the table. We also have +# a span for all other secondary indexes with the same config at the end. +# We get two more spans that subdivide the primary index as well. One before the +# start of the first partition and one after the end of the second partition. This +# second span covers the remaining primary index. Both these spans have num_voters +# set to 6, as that's what we did above. +translate database=db table=t +---- +/Table/53{-/1} num_replicas=7 num_voters=5 +/Table/53/1{-/1} num_replicas=7 num_voters=6 +/Table/53/1/{1-2} global_reads=true num_replicas=7 num_voters=5 +/Table/53/1/{2-3} global_reads=true num_replicas=7 num_voters=5 +/Table/53/1/{3-4} ttl_seconds=5 num_replicas=7 num_voters=3 +/Table/53/1/{4-5} ttl_seconds=5 num_replicas=7 num_voters=3 +/Table/53/{1/5-2} num_replicas=7 num_voters=6 +/Table/5{3/2-4} num_replicas=7 num_voters=5 + +# Discard the table's zone configuration. This essentially means that the table +# has a "placeholder" zone config (to capture partition subzone configs). +exec-sql +ALTER TABLE db.t CONFIGURE ZONE DISCARD +---- + +# The expectation here is very similar to above, except the spans before the +# primary index and for secondary indexes now inherit num_voters from the +# database. This is because we removed the explicit zone configuration on the +# table above (which is where num_voters=5 was coming from). +translate database=db table=t +---- +/Table/53{-/1} num_replicas=7 +/Table/53/1{-/1} num_replicas=7 num_voters=6 +/Table/53/1/{1-2} global_reads=true num_replicas=7 +/Table/53/1/{2-3} global_reads=true num_replicas=7 +/Table/53/1/{3-4} ttl_seconds=5 num_replicas=7 num_voters=3 +/Table/53/1/{4-5} ttl_seconds=5 num_replicas=7 num_voters=3 +/Table/53/{1/5-2} num_replicas=7 num_voters=6 +/Table/5{3/2-4} num_replicas=7 diff --git a/pkg/config/zonepb/BUILD.bazel b/pkg/config/zonepb/BUILD.bazel index c3240f84a361..3cc68c6ca75f 100644 --- a/pkg/config/zonepb/BUILD.bazel +++ b/pkg/config/zonepb/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//pkg/base", "//pkg/keys", "//pkg/roachpb:with-mocks", + "//pkg/sql/catalog/descpb", "//pkg/sql/opt/cat", "//pkg/sql/sem/tree", "//pkg/util/log", diff --git a/pkg/config/zonepb/zone.go b/pkg/config/zonepb/zone.go index 55657f3a0dbf..fab065a2dbf8 100644 --- a/pkg/config/zonepb/zone.go +++ b/pkg/config/zonepb/zone.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -26,20 +27,35 @@ import ( "github.com/gogo/protobuf/proto" ) +// NamedZone is a custom type for names used to reference ranges outside the +// SQL keyspace by zone configurations. +type NamedZone string + // Several ranges outside of the SQL keyspace are given special names so they // can be targeted by zone configs. const ( - DefaultZoneName = "default" - LivenessZoneName = "liveness" - MetaZoneName = "meta" - SystemZoneName = "system" - TimeseriesZoneName = "timeseries" - TenantsZoneName = "tenants" + DefaultZoneName NamedZone = "default" + LivenessZoneName NamedZone = "liveness" + MetaZoneName NamedZone = "meta" + SystemZoneName NamedZone = "system" + TimeseriesZoneName NamedZone = "timeseries" + TenantsZoneName NamedZone = "tenants" ) +// NamedZonesList is a list of all named zones that reference ranges +// outside the SQL keyspace which can be referenced by zone configurations. +var NamedZonesList = [...]NamedZone{ + DefaultZoneName, + LivenessZoneName, + MetaZoneName, + SystemZoneName, + TimeseriesZoneName, + TenantsZoneName, +} + // NamedZones maps named zones to their pseudo-table ID that can be used to // install an entry into the system.zones table. -var NamedZones = map[string]uint32{ +var NamedZones = map[NamedZone]uint32{ DefaultZoneName: keys.RootNamespaceID, LivenessZoneName: keys.LivenessRangesID, MetaZoneName: keys.MetaRangesID, @@ -50,14 +66,21 @@ var NamedZones = map[string]uint32{ // NamedZonesByID is the inverse of NamedZones: it maps pseudo-table IDs to // their zone names. -var NamedZonesByID = func() map[uint32]string { - out := map[uint32]string{} +var NamedZonesByID = func() map[uint32]NamedZone { + out := map[uint32]NamedZone{} for name, id := range NamedZones { out[id] = name } return out }() +// IsNamedZoneID returns true if the given ID is one of the pseudo-table IDs +// that maps to named zones. +func IsNamedZoneID(id descpb.ID) bool { + _, ok := NamedZonesByID[(uint32(id))] + return ok +} + // MultiRegionZoneConfigFields are the fields on a zone configuration which // may be set by the system for multi-region objects". var MultiRegionZoneConfigFields = []tree.Name{ @@ -120,10 +143,10 @@ func ResolveZoneSpecifier( // - a database name; // - a table or index name. if zs.NamedZone != "" { - if zs.NamedZone == DefaultZoneName { + if NamedZone(zs.NamedZone) == DefaultZoneName { return keys.RootNamespaceID, nil } - if id, ok := NamedZones[string(zs.NamedZone)]; ok { + if id, ok := NamedZones[NamedZone(zs.NamedZone)]; ok { return id, nil } return 0, fmt.Errorf("%q is not a built-in zone", string(zs.NamedZone)) @@ -1136,7 +1159,7 @@ func (z *ZoneConfig) EnsureFullyHydrated() error { // AsSpanConfig converts a fully hydrated zone configuration to an equivalent // SpanConfig. It fatals if the zone config hasn't been fully hydrated (fields // are expected to have been cascaded through parent zone configs). -func (z *ZoneConfig) AsSpanConfig() roachpb.SpanConfig { +func (z ZoneConfig) AsSpanConfig() roachpb.SpanConfig { spanConfig, err := z.toSpanConfig() if err != nil { log.Fatalf(context.Background(), "%v", err) @@ -1195,22 +1218,38 @@ func (z *ZoneConfig) toSpanConfig() (roachpb.SpanConfig, error) { return constraintsConjunction, nil } - sc.Constraints = make([]roachpb.ConstraintsConjunction, len(z.Constraints)) - sc.Constraints, err = toSpanConfigConstraintsConjunction(z.Constraints) - if err != nil { - return roachpb.SpanConfig{}, err - } - sc.VoterConstraints, err = toSpanConfigConstraintsConjunction(z.VoterConstraints) - if err != nil { - return roachpb.SpanConfig{}, err + if len(z.Constraints) != 0 { + sc.Constraints = make([]roachpb.ConstraintsConjunction, len(z.Constraints)) + sc.Constraints, err = toSpanConfigConstraintsConjunction(z.Constraints) + if err != nil { + return roachpb.SpanConfig{}, err + } } - - sc.LeasePreferences = make([]roachpb.LeasePreference, len(z.LeasePreferences)) - for i, leasePreference := range z.LeasePreferences { - sc.LeasePreferences[i].Constraints, err = toSpanConfigConstraints(leasePreference.Constraints) + if len(z.VoterConstraints) != 0 { + sc.VoterConstraints, err = toSpanConfigConstraintsConjunction(z.VoterConstraints) if err != nil { return roachpb.SpanConfig{}, err } } + + if len(z.LeasePreferences) != 0 { + sc.LeasePreferences = make([]roachpb.LeasePreference, len(z.LeasePreferences)) + for i, leasePreference := range z.LeasePreferences { + sc.LeasePreferences[i].Constraints, err = toSpanConfigConstraints(leasePreference.Constraints) + if err != nil { + return roachpb.SpanConfig{}, err + } + } + } return sc, nil } + +func init() { + if len(NamedZonesList) != len(NamedZones) { + panic(fmt.Errorf( + "NamedZonesList (%d) and NamedZones (%d) should have the same number of entries", + len(NamedZones), + len(NamedZonesList), + )) + } +} diff --git a/pkg/config/zonepb/zone_test.go b/pkg/config/zonepb/zone_test.go index 2ac12278f5b2..3342c8cd9953 100644 --- a/pkg/config/zonepb/zone_test.go +++ b/pkg/config/zonepb/zone_test.go @@ -975,13 +975,13 @@ func TestZoneSpecifiers(t *testing.T) { // Simulate exactly two named zones: one named default and one named carl. // N.B. DefaultZoneName must always exist in the mapping; it is treated // specially so that it always appears first in the lookup path. - defer func(old map[string]uint32) { NamedZones = old }(NamedZones) - NamedZones = map[string]uint32{ + defer func(old map[NamedZone]uint32) { NamedZones = old }(NamedZones) + NamedZones = map[NamedZone]uint32{ DefaultZoneName: 0, "carl": 42, } - defer func(old map[uint32]string) { NamedZonesByID = old }(NamedZonesByID) - NamedZonesByID = map[uint32]string{ + defer func(old map[uint32]NamedZone) { NamedZonesByID = old }(NamedZonesByID) + NamedZonesByID = map[uint32]NamedZone{ 0: DefaultZoneName, 42: "carl", } @@ -1188,12 +1188,9 @@ func TestZoneConfigToSpanConfigConversion(t *testing.T) { GCPolicy: roachpb.GCPolicy{ TTLSeconds: 2400, }, - GlobalReads: false, - NumVoters: 0, - NumReplicas: 3, - Constraints: []roachpb.ConstraintsConjunction{}, - VoterConstraints: []roachpb.ConstraintsConjunction{}, - LeasePreferences: []roachpb.LeasePreference{}, + GlobalReads: false, + NumVoters: 0, + NumReplicas: 3, }, }, { @@ -1213,12 +1210,9 @@ func TestZoneConfigToSpanConfigConversion(t *testing.T) { GCPolicy: roachpb.GCPolicy{ TTLSeconds: 2400, }, - GlobalReads: true, - NumVoters: 0, - NumReplicas: 3, - Constraints: []roachpb.ConstraintsConjunction{}, - VoterConstraints: []roachpb.ConstraintsConjunction{}, - LeasePreferences: []roachpb.LeasePreference{}, + GlobalReads: true, + NumVoters: 0, + NumReplicas: 3, }, }, { @@ -1238,12 +1232,9 @@ func TestZoneConfigToSpanConfigConversion(t *testing.T) { GCPolicy: roachpb.GCPolicy{ TTLSeconds: 2400, }, - GlobalReads: false, - NumVoters: 0, - NumReplicas: 3, - Constraints: []roachpb.ConstraintsConjunction{}, - VoterConstraints: []roachpb.ConstraintsConjunction{}, - LeasePreferences: []roachpb.LeasePreference{}, + GlobalReads: false, + NumVoters: 0, + NumReplicas: 3, }, }, { @@ -1312,8 +1303,6 @@ func TestZoneConfigToSpanConfigConversion(t *testing.T) { }, }, }, - VoterConstraints: []roachpb.ConstraintsConjunction{}, - LeasePreferences: []roachpb.LeasePreference{}, }, }, { @@ -1347,7 +1336,6 @@ func TestZoneConfigToSpanConfigConversion(t *testing.T) { GlobalReads: false, NumVoters: 0, NumReplicas: 3, - Constraints: []roachpb.ConstraintsConjunction{}, VoterConstraints: []roachpb.ConstraintsConjunction{ { Constraints: []roachpb.Constraint{ @@ -1360,7 +1348,6 @@ func TestZoneConfigToSpanConfigConversion(t *testing.T) { }, }, }, - LeasePreferences: []roachpb.LeasePreference{}, }, }, { @@ -1396,11 +1383,9 @@ func TestZoneConfigToSpanConfigConversion(t *testing.T) { GCPolicy: roachpb.GCPolicy{ TTLSeconds: 2400, }, - GlobalReads: false, - NumVoters: 0, - NumReplicas: 3, - Constraints: []roachpb.ConstraintsConjunction{}, - VoterConstraints: []roachpb.ConstraintsConjunction{}, + GlobalReads: false, + NumVoters: 0, + NumReplicas: 3, LeasePreferences: []roachpb.LeasePreference{ { Constraints: []roachpb.Constraint{ diff --git a/pkg/keys/spans.go b/pkg/keys/spans.go index f7de69b62043..d611b2ee8a02 100644 --- a/pkg/keys/spans.go +++ b/pkg/keys/spans.go @@ -31,6 +31,9 @@ var ( // NodeLivenessSpan holds the liveness records for nodes in the cluster. NodeLivenessSpan = roachpb.Span{Key: NodeLivenessPrefix, EndKey: NodeLivenessKeyMax} + // TimeseriesSpan holds all the timeseries data in the cluster. + TimeseriesSpan = roachpb.Span{Key: TimeseriesPrefix, EndKey: TimeseriesKeyMax} + // SystemConfigSpan is the range of system objects which will be gossiped. SystemConfigSpan = roachpb.Span{Key: SystemConfigSplitKey, EndKey: SystemConfigTableDataMax} diff --git a/pkg/roachpb/span_config.go b/pkg/roachpb/span_config.go index 546c137bb844..f3ce2a73f475 100644 --- a/pkg/roachpb/span_config.go +++ b/pkg/roachpb/span_config.go @@ -57,6 +57,8 @@ func (s *SpanConfig) TTL() time.Duration { // GetNumVoters returns the number of voting replicas as defined in the // span config. +// TODO(arul): We can get rid of this now that we're correctly populating +// numVoters when going from ZoneConfigs -> SpanConfigs. func (s *SpanConfig) GetNumVoters() int32 { if s.NumVoters != 0 { return s.NumVoters diff --git a/pkg/server/BUILD.bazel b/pkg/server/BUILD.bazel index 25794944c0b6..867dce5b597f 100644 --- a/pkg/server/BUILD.bazel +++ b/pkg/server/BUILD.bazel @@ -114,6 +114,7 @@ go_library( "//pkg/spanconfig/spanconfigjob", "//pkg/spanconfig/spanconfigkvaccessor", "//pkg/spanconfig/spanconfigmanager", + "//pkg/spanconfig/spanconfigsqltranslator", "//pkg/sql", "//pkg/sql/catalog/bootstrap", "//pkg/sql/catalog/catalogkeys", diff --git a/pkg/server/server_sql.go b/pkg/server/server_sql.go index 94a6b5bc1e3f..699f3aa4da05 100644 --- a/pkg/server/server_sql.go +++ b/pkg/server/server_sql.go @@ -52,6 +52,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqltranslator" "github.com/cockroachdb/cockroach/pkg/sql" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkeys" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" @@ -840,6 +841,7 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) { // Instantiate a span config manager. If we're the host tenant we'll // only do it if COCKROACH_EXPERIMENTAL_SPAN_CONFIGS is set. spanConfigKnobs, _ := cfg.TestingKnobs.SpanConfig.(*spanconfig.TestingKnobs) + sqlTranslator := spanconfigsqltranslator.New(execCfg, codec) spanConfigMgr = spanconfigmanager.New( cfg.db, jobRegistry, @@ -847,6 +849,7 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) { cfg.stopper, cfg.Settings, cfg.spanConfigAccessor, + sqlTranslator, spanConfigKnobs, ) execCfg.SpanConfigReconciliationJobDeps = spanConfigMgr diff --git a/pkg/server/testserver.go b/pkg/server/testserver.go index 55154f6a6970..f4710bbf7a5c 100644 --- a/pkg/server/testserver.go +++ b/pkg/server/testserver.go @@ -952,6 +952,16 @@ func (ts *TestServer) SpanConfigAccessor() interface{} { return ts.Server.node.spanConfigAccessor } +// SpanConfigSQLTranslator is part of TestServerInterface. +func (ts *TestServer) SpanConfigSQLTranslator() interface{} { + if ts.sqlServer.spanconfigMgr == nil { + panic( + "span config manager uninitialized; see EnableSpanConfigs testing knob to use span configs", + ) + } + return ts.sqlServer.spanconfigMgr.SQLTranslator +} + // SQLServer is part of TestServerInterface. func (ts *TestServer) SQLServer() interface{} { return ts.PGServer().SQLServer diff --git a/pkg/spanconfig/BUILD.bazel b/pkg/spanconfig/BUILD.bazel index e949d5d82f05..6f3d8fc5a0b0 100644 --- a/pkg/spanconfig/BUILD.bazel +++ b/pkg/spanconfig/BUILD.bazel @@ -10,6 +10,9 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/base", + "//pkg/keys", "//pkg/roachpb:with-mocks", + "//pkg/sql/catalog/descpb", + "//pkg/util/hlc", ], ) diff --git a/pkg/spanconfig/spanconfig.go b/pkg/spanconfig/spanconfig.go index 2917bdf889da..01b07a082167 100644 --- a/pkg/spanconfig/spanconfig.go +++ b/pkg/spanconfig/spanconfig.go @@ -13,7 +13,10 @@ package spanconfig import ( "context" + "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/util/hlc" ) // KVAccessor mediates access to KV span configurations pertaining to a given @@ -33,6 +36,47 @@ type KVAccessor interface { UpdateSpanConfigEntries(ctx context.Context, toDelete []roachpb.Span, toUpsert []roachpb.SpanConfigEntry) error } +// SQLTranslator translates SQL descriptors and their corresponding zone +// configurations to constituent spans and span configurations. +// +// Concretely, for the following zone configuration hierarchy: +// CREATE DATABASE db; +// CREATE TABLE db.t1(); +// ALTER DATABASE db CONFIGURE ZONE USING num_replicas=7; +// ALTER TABLE db.t1 CONFIGURE ZONE USING num_voters=5; +// The SQLTranslator produces the following translation (represented as a diff +// against RANGE DEFAULT for brevity): +// Table/5{3-4} num_replicas=7 num_voters=5 +type SQLTranslator interface { + // Translate generates the implied span configuration state given a list of + // Translate generates the span configuration state given a list of + // {descriptor, named zone} IDs. No entry is returned for an ID if it doesn't + // exist or has been dropped. The timestamp at which the translation is valid + // is also returned. + // + // For every ID we first descend the zone configuration hierarchy with the ID + // as the root to accumulate IDs of all leaf objects. Leaf objects are tables + // and named zones (other than RANGE DEFAULT) which have actual span + // configurations associated with them (as opposed to non-leaf nodes that only + // serve to hold zone configurations for inheritance purposes). Then, for + // for every one of these accumulated IDs, we generate + // tuples by following up the inheritance chain to fully hydrate the span + // configuration. This also accounts for and negotiates subzone spans. + Translate(ctx context.Context, ids descpb.IDs) ([]roachpb.SpanConfigEntry, hlc.Timestamp, error) +} + +// FullTranslate translates the entire SQL zone configuration state to the +// span configuration state. The timestamp at which such a translation is valid +// is also returned. +func FullTranslate( + ctx context.Context, s SQLTranslator, +) ([]roachpb.SpanConfigEntry, hlc.Timestamp, error) { + // As RANGE DEFAULT is the root of all zone configurations (including + // other named zones for the system tenant), we can construct the entire + // span configuration state by starting from RANGE DEFAULT. + return s.Translate(ctx, descpb.IDs{keys.RootNamespaceID}) +} + // ReconciliationDependencies captures what's needed by the span config // reconciliation job to perform its task. The job is responsible for // reconciling a tenant's zone configurations with the clusters span @@ -40,11 +84,13 @@ type KVAccessor interface { type ReconciliationDependencies interface { KVAccessor - // TODO(irfansharif): We'll also want access to a "SQLWatcher", something - // that watches for changes to system.{descriptor,zones} and be responsible - // for generating corresponding span config updates. Put together, the - // reconciliation job will react to these updates by installing them into KV - // through the KVAccessor. + SQLTranslator + + // TODO(arul): We'll also want access to a "SQLWatcher", something that + // watches for changes to system.{descriptors, zones} to feed IDs to the + // SQLTranslator. These interfaces will be used by the "Reconciler to perform + // full/partial reconciliation, checkpoint the span config job, and update KV + // with the tenants span config state. } // Store is a data structure used to store spans and their corresponding diff --git a/pkg/spanconfig/spanconfigmanager/manager.go b/pkg/spanconfig/spanconfigmanager/manager.go index 8756e8fcf9c6..defe3b3f7cc5 100644 --- a/pkg/spanconfig/spanconfigmanager/manager.go +++ b/pkg/spanconfig/spanconfigmanager/manager.go @@ -60,6 +60,7 @@ type Manager struct { knobs *spanconfig.TestingKnobs spanconfig.KVAccessor + spanconfig.SQLTranslator } var _ spanconfig.ReconciliationDependencies = &Manager{} @@ -72,19 +73,21 @@ func New( stopper *stop.Stopper, settings *cluster.Settings, kvAccessor spanconfig.KVAccessor, + sqlTranslator spanconfig.SQLTranslator, knobs *spanconfig.TestingKnobs, ) *Manager { if knobs == nil { knobs = &spanconfig.TestingKnobs{} } return &Manager{ - db: db, - jr: jr, - ie: ie, - stopper: stopper, - settings: settings, - knobs: knobs, - KVAccessor: kvAccessor, + db: db, + jr: jr, + ie: ie, + stopper: stopper, + settings: settings, + KVAccessor: kvAccessor, + SQLTranslator: sqlTranslator, + knobs: knobs, } } diff --git a/pkg/spanconfig/spanconfigmanager/manager_test.go b/pkg/spanconfig/spanconfigmanager/manager_test.go index 7849796c6709..a8b0a7bd7ab2 100644 --- a/pkg/spanconfig/spanconfigmanager/manager_test.go +++ b/pkg/spanconfig/spanconfigmanager/manager_test.go @@ -74,6 +74,7 @@ func TestManagerConcurrentJobCreation(t *testing.T) { ts.Stopper(), ts.ClusterSettings(), ts.SpanConfigAccessor().(spanconfig.KVAccessor), + ts.SpanConfigSQLTranslator().(spanconfig.SQLTranslator), &spanconfig.TestingKnobs{ ManagerCreatedJobInterceptor: func(jobI interface{}) { job := jobI.(*jobs.Job) @@ -161,6 +162,7 @@ func TestManagerStartsJobIfFailed(t *testing.T) { ts.Stopper(), ts.ClusterSettings(), ts.SpanConfigAccessor().(spanconfig.KVAccessor), + ts.SpanConfigSQLTranslator().(spanconfig.SQLTranslator), &spanconfig.TestingKnobs{ ManagerAfterCheckedReconciliationJobExistsInterceptor: func(exists bool) { require.False(t, exists) @@ -235,6 +237,7 @@ func TestManagerCheckJobConditions(t *testing.T) { ts.Stopper(), ts.ClusterSettings(), ts.SpanConfigAccessor().(spanconfig.KVAccessor), + ts.SpanConfigSQLTranslator().(spanconfig.SQLTranslator), &spanconfig.TestingKnobs{ ManagerDisableJobCreation: true, ManagerCheckJobInterceptor: func() { diff --git a/pkg/spanconfig/spanconfigsqltranslator/BUILD.bazel b/pkg/spanconfig/spanconfigsqltranslator/BUILD.bazel new file mode 100644 index 000000000000..a81b0ccc81ba --- /dev/null +++ b/pkg/spanconfig/spanconfigsqltranslator/BUILD.bazel @@ -0,0 +1,22 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "spanconfigsqltranslator", + srcs = ["sql_translator.go"], + importpath = "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqltranslator", + visibility = ["//visibility:public"], + deps = [ + "//pkg/config/zonepb", + "//pkg/keys", + "//pkg/kv", + "//pkg/roachpb:with-mocks", + "//pkg/spanconfig", + "//pkg/sql", + "//pkg/sql/catalog", + "//pkg/sql/catalog/descpb", + "//pkg/sql/catalog/descs", + "//pkg/sql/sem/tree", + "//pkg/util/hlc", + "@com_github_cockroachdb_errors//:errors", + ], +) diff --git a/pkg/spanconfig/spanconfigsqltranslator/sql_translator.go b/pkg/spanconfig/spanconfigsqltranslator/sql_translator.go new file mode 100644 index 000000000000..3bd6e6727df0 --- /dev/null +++ b/pkg/spanconfig/spanconfigsqltranslator/sql_translator.go @@ -0,0 +1,385 @@ +// Copyright 2021 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package spanconfigsqltranslator + +import ( + "context" + + "github.com/cockroachdb/cockroach/pkg/config/zonepb" + "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/kv" + "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/sql" + "github.com/cockroachdb/cockroach/pkg/sql/catalog" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/errors" +) + +// SQLTranslator implements the spanconfig.SQLTranslator interface. +var _ spanconfig.SQLTranslator = &SQLTranslator{} + +// SQLTranslator is the concrete implementation of spanconfig.SQLTranslator. +type SQLTranslator struct { + execCfg *sql.ExecutorConfig + codec keys.SQLCodec +} + +// New constructs and returns a SQLTranslator. +func New(execCfg *sql.ExecutorConfig, codec keys.SQLCodec) *SQLTranslator { + return &SQLTranslator{ + execCfg: execCfg, + codec: codec, + } +} + +// Translate is part of the spanconfig.SQLTranslator interface. +func (s *SQLTranslator) Translate( + ctx context.Context, ids descpb.IDs, +) ([]roachpb.SpanConfigEntry, hlc.Timestamp, error) { + var entries []roachpb.SpanConfigEntry + // txn used to translate the IDs, so that we can get its commit timestamp + // later. + var translateTxn *kv.Txn + if err := sql.DescsTxn(ctx, s.execCfg, func( + ctx context.Context, txn *kv.Txn, descsCol *descs.Collection, + ) error { + // We're in a retryable closure, so clear any entries from previous + // attempts. + entries = make([]roachpb.SpanConfigEntry, 0) + + // For every ID we want to translate, first expand it to descendant leaf + // IDs that have span configurations associated for them. We also + // de-duplicate leaf IDs so as to not generate redundant entries. + seen := make(map[descpb.ID]struct{}) + var leafIDs descpb.IDs + for _, id := range ids { + descendantLeafIDs, err := s.findDescendantLeafIDs(ctx, id, txn, descsCol) + if err != nil { + return err + } + for _, descendantLeafID := range descendantLeafIDs { + if _, found := seen[descendantLeafID]; !found { + seen[descendantLeafID] = struct{}{} + leafIDs = append(leafIDs, descendantLeafID) + } + } + } + + // For every leaf ID, which has been de-duplicated, generate span + // configurations. + for _, leafID := range leafIDs { + translatedEntries, err := s.generateSpanConfigurations(ctx, leafID, txn, descsCol) + if err != nil { + return err + } + entries = append(entries, translatedEntries...) + } + translateTxn = txn + return nil + }); err != nil { + return nil, hlc.Timestamp{}, err + } + + return entries, translateTxn.CommitTimestamp(), nil +} + +// generateSpanConfigurations generates the span configurations for the given +// ID. The ID must belong to an object that has a span configuration associated +// with it, i.e, it should either belong to a table or a named zone. +func (s *SQLTranslator) generateSpanConfigurations( + ctx context.Context, id descpb.ID, txn *kv.Txn, descsCol *descs.Collection, +) (entries []roachpb.SpanConfigEntry, err error) { + if zonepb.IsNamedZoneID(id) { + return s.generateSpanConfigurationsForNamedZone(ctx, txn, id) + } + + // We're dealing with a SQL object. + desc, err := descsCol.GetImmutableDescriptorByID(ctx, txn, id, tree.CommonLookupFlags{ + // We can (do) generate span configurations for dropped/offline tables. + IncludeDropped: true, + IncludeOffline: true, + }) + if err != nil { + if errors.Is(err, catalog.ErrDescriptorNotFound) { + // The descriptor has been deleted. Nothing to do here. + return nil, nil + } + return nil, err + } + + if desc.DescriptorType() != catalog.Table { + return nil, errors.AssertionFailedf( + "can only generate span configurations for tables, but got %s", desc.DescriptorType(), + ) + } + + return s.generateSpanConfigurationsForTable(ctx, txn, desc) +} + +// generateSpanConfigurationsForNamedZone expects an ID corresponding to a named +// zone and generates the span configurations for it. +func (s *SQLTranslator) generateSpanConfigurationsForNamedZone( + ctx context.Context, txn *kv.Txn, id descpb.ID, +) ([]roachpb.SpanConfigEntry, error) { + name, ok := zonepb.NamedZonesByID[uint32(id)] + if !ok { + return nil, errors.AssertionFailedf("id %d does not belong to a named zone", id) + } + + // Named zones other than RANGE DEFAULT are not a thing for secondary tenants. + if !s.codec.ForSystemTenant() && name != zonepb.DefaultZoneName { + return nil, + errors.AssertionFailedf("secondary tenants do not have the notion of %s named zone", name) + } + + var spans []roachpb.Span + switch name { + case zonepb.DefaultZoneName: // nothing to do. + case zonepb.MetaZoneName: + spans = append(spans, roachpb.Span{Key: keys.Meta1Span.Key, EndKey: keys.NodeLivenessSpan.Key}) + case zonepb.LivenessZoneName: + spans = append(spans, keys.NodeLivenessSpan) + case zonepb.TimeseriesZoneName: + spans = append(spans, keys.TimeseriesSpan) + case zonepb.SystemZoneName: + // Add spans for the system range without the timeseries and + // liveness ranges, which are individually captured above. + // + // Note that the NodeLivenessSpan sorts before the rest of the system + // keyspace, so the first span here starts at the end of the + // NodeLivenessSpan. + spans = append(spans, roachpb.Span{ + Key: keys.NodeLivenessSpan.EndKey, + EndKey: keys.TimeseriesSpan.Key, + }) + spans = append(spans, roachpb.Span{ + Key: keys.TimeseriesSpan.EndKey, + EndKey: keys.SystemMax, + }) + case zonepb.TenantsZoneName: // nothing to do. + default: + return nil, errors.AssertionFailedf("unknown named zone config %s", name) + } + + zoneConfig, err := sql.GetHydratedZoneConfigForNamedZone(ctx, txn, s.codec, name) + if err != nil { + return nil, err + } + spanConfig := zoneConfig.AsSpanConfig() + + var entries []roachpb.SpanConfigEntry + for _, span := range spans { + entries = append(entries, roachpb.SpanConfigEntry{ + Span: span, + Config: spanConfig, + }) + } + return entries, nil +} + +// generateSpanConfigurationsForTable generates the span configurations +// corresponding to the given tableID. It uses a transactional view of +// system.zones and system.descriptors to do so. +func (s *SQLTranslator) generateSpanConfigurationsForTable( + ctx context.Context, txn *kv.Txn, desc catalog.Descriptor, +) ([]roachpb.SpanConfigEntry, error) { + if desc.DescriptorType() != catalog.Table { + return nil, errors.AssertionFailedf( + "expected table descriptor, but got descriptor of type %s", desc.DescriptorType(), + ) + } + zone, err := sql.GetHydratedZoneConfigForTable(ctx, txn, s.codec, desc.GetID()) + if err != nil { + return nil, err + } + spanConfig := zone.AsSpanConfig() + + ret := make([]roachpb.SpanConfigEntry, 0) + prevEndKey := s.codec.TablePrefix(uint32(desc.GetID())) + for i := range zone.SubzoneSpans { + // We need to prepend the tablePrefix to the spans stored inside the + // SubzoneSpans field because we store the stripped version there for + // historical reasons. + span := roachpb.Span{ + Key: append(s.codec.TablePrefix(uint32(desc.GetID())), zone.SubzoneSpans[i].Key...), + EndKey: append(s.codec.TablePrefix(uint32(desc.GetID())), zone.SubzoneSpans[i].EndKey...), + } + + { + // The zone config code sets the EndKey to be nil before storing the + // proto if it is equal to `Key.PrefixEnd()`, so we bring it back if + // required. + if zone.SubzoneSpans[i].EndKey == nil { + span.EndKey = span.Key.PrefixEnd() + } + } + + // If there is a "hole" in the spans covered by the subzones array we fill + // it using the parent zone configuration. + if !prevEndKey.Equal(span.Key) { + ret = append(ret, + roachpb.SpanConfigEntry{ + Span: roachpb.Span{Key: prevEndKey, EndKey: span.Key}, + Config: spanConfig, + }, + ) + } + + // Add an entry for the subzone. + subzoneSpanConfig := zone.Subzones[zone.SubzoneSpans[i].SubzoneIndex].Config.AsSpanConfig() + ret = append(ret, + roachpb.SpanConfigEntry{ + Span: roachpb.Span{Key: span.Key, EndKey: span.EndKey}, + Config: subzoneSpanConfig, + }, + ) + + prevEndKey = span.EndKey + } + + // If the last subzone span doesn't cover the entire table's keyspace then we + // cover the remaining key range with the table's zone configuration. + if !prevEndKey.Equal(s.codec.TablePrefix(uint32(desc.GetID())).PrefixEnd()) { + ret = append(ret, + roachpb.SpanConfigEntry{ + Span: roachpb.Span{Key: prevEndKey, EndKey: s.codec.TablePrefix(uint32(desc.GetID())).PrefixEnd()}, + Config: spanConfig, + }, + ) + } + return ret, nil +} + +// findDescendantLeafIDs finds all leaf IDs below the given ID in the zone +// configuration hierarchy. Leaf IDs are either table IDs or named zone IDs +// (other than RANGE DEFAULT). +func (s *SQLTranslator) findDescendantLeafIDs( + ctx context.Context, id descpb.ID, txn *kv.Txn, descsCol *descs.Collection, +) (descpb.IDs, error) { + if zonepb.IsNamedZoneID(id) { + return s.findDescendantLeafIDsForNamedZone(ctx, id, txn, descsCol) + } + // We're dealing with a SQL Object here. + return s.findDescendantLeafIDsForDescriptor(ctx, id, txn, descsCol) +} + +// findDescendantLeafIDsForDescriptor finds all leaf object IDs below the given +// descriptor ID in the zone configuration hierarchy. Based on the descriptor +// type, these are: +// - Database: IDs of all tables inside the database. +// - Table: ID of the table itself. +// - Schema/Type: Nothing, as schemas/types do not carry zone configurations and +// are not part of the zone configuration hierarchy. +func (s *SQLTranslator) findDescendantLeafIDsForDescriptor( + ctx context.Context, id descpb.ID, txn *kv.Txn, descsCol *descs.Collection, +) (descpb.IDs, error) { + desc, err := descsCol.GetImmutableDescriptorByID(ctx, txn, id, tree.CommonLookupFlags{ + // It is reasonable for us to need to traverse zone configuration + // hierarchies for dropped/offline descriptors. For example, when reacting + // to a SQLWatcher event when a table is moved from PUBLIC to OFFLINE. + IncludeDropped: true, + IncludeOffline: true, + }) + if err != nil { + if errors.Is(err, catalog.ErrDescriptorNotFound) { + // The descriptor has been deleted. Nothing to do here. + return nil, nil + } + return nil, err + } + + switch desc.DescriptorType() { + case catalog.Type, catalog.Schema: + // There is nothing to do for {Type, Schema} descriptors as they are not + // part of the zone configuration hierarchy. + return nil, nil + case catalog.Table: + // Tables are leaf objects in the zone configuration hierarchy, so simply + // return the ID. + return descpb.IDs{id}, nil + case catalog.Database: + // Fallthrough. + default: + return nil, errors.AssertionFailedf("unknown descriptor type: %s", desc.DescriptorType()) + } + + // There's nothing for us to do if the descriptor is offline or has been + // dropped. + if desc.Offline() || desc.Dropped() { + return nil, nil + } + + // Expand the database descriptor to all the tables inside it and return their + // IDs. + tables, err := descsCol.GetAllTableDescriptorsInDatabase(ctx, txn, desc.GetID()) + if err != nil { + return nil, err + } + ret := make(descpb.IDs, 0, len(tables)) + for _, table := range tables { + ret = append(ret, table.GetID()) + } + return ret, nil +} + +// findDescendantLeafIDsForNamedZone finds all leaf IDs below the given named +// zone ID in the zone configuration hierarchy. +// Depending on the named zone, these are: +// - RANGE DEFAULT: All tables (and named zones iff system tenant). +// - Any other named zone: ID of the named zone itself. +func (s *SQLTranslator) findDescendantLeafIDsForNamedZone( + ctx context.Context, id descpb.ID, txn *kv.Txn, descsCol *descs.Collection, +) (descpb.IDs, error) { + name, ok := zonepb.NamedZonesByID[uint32(id)] + if !ok { + return nil, errors.AssertionFailedf("id %d does not belong to a named zone", id) + } + + if name != zonepb.DefaultZoneName { + // No IDs lie below named zones other than RANGE DEFAULT in the zone config + // hierarchy, so simply return the named zone ID. + return descpb.IDs{id}, nil + } + + // A change to RANGE DEFAULT translates to every SQL object of the tenant. + databases, err := descsCol.GetAllDatabaseDescriptors(ctx, txn) + if err != nil { + return nil, err + } + var descendantIDs descpb.IDs + for _, dbDesc := range databases { + tableIDs, err := s.findDescendantLeafIDsForDescriptor( + ctx, dbDesc.GetID(), txn, descsCol, + ) + if err != nil { + return nil, err + } + descendantIDs = append(descendantIDs, tableIDs...) + } + + // All named zones (other than RANGE DEFAULT itself, ofcourse) inherit from + // RANGE DEFAULT. + // NB: Only the system tenant has named zones other than RANGE DEFAULT. + if s.codec.ForSystemTenant() { + for _, namedZone := range zonepb.NamedZonesList { + // Add an entry for all named zones bar RANGE DEFAULT. + if namedZone == zonepb.DefaultZoneName { + continue + } + descendantIDs = append(descendantIDs, descpb.ID(zonepb.NamedZones[namedZone])) + } + } + return descendantIDs, nil +} diff --git a/pkg/sql/catalog/descs/collection.go b/pkg/sql/catalog/descs/collection.go index b482defc6906..49e63c1b17a8 100644 --- a/pkg/sql/catalog/descs/collection.go +++ b/pkg/sql/catalog/descs/collection.go @@ -15,6 +15,7 @@ package descs import ( "bytes" "context" + "fmt" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" @@ -27,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog/hydratedtables" "github.com/cockroachdb/cockroach/pkg/sql/catalog/lease" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/sqlerrors" "github.com/cockroachdb/cockroach/pkg/sql/sqlliveness" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -278,6 +280,38 @@ func (tc *Collection) GetAllDatabaseDescriptors( return tc.kv.getAllDatabaseDescriptors(ctx, txn) } +// GetAllTableDescriptorsInDatabase returns all the table descriptors visible to +// the transaction under the database with the given ID. It first checks the +// collection's cached descriptors before defaulting to a key-value scan, if +// necessary. +func (tc *Collection) GetAllTableDescriptorsInDatabase( + ctx context.Context, txn *kv.Txn, dbID descpb.ID, +) ([]catalog.TableDescriptor, error) { + // Ensure the given ID does indeed belong to a database. + found, _, err := tc.getDatabaseByID(ctx, txn, dbID, tree.DatabaseLookupFlags{ + AvoidCached: false, + }) + if err != nil { + return nil, err + } + if !found { + return nil, sqlerrors.NewUndefinedDatabaseError(fmt.Sprintf("[%d]", dbID)) + } + descs, err := tc.GetAllDescriptors(ctx, txn) + if err != nil { + return nil, err + } + var ret []catalog.TableDescriptor + for _, desc := range descs { + if desc.GetParentID() == dbID { + if table, ok := desc.(catalog.TableDescriptor); ok { + ret = append(ret, table) + } + } + } + return ret, nil +} + // GetSchemasForDatabase returns the schemas for a given database // visible by the transaction. This uses the schema cache locally // if possible, or else performs a scan on kv. diff --git a/pkg/sql/show_zone_config.go b/pkg/sql/show_zone_config.go index 16868941d533..3cec49eb5e6a 100644 --- a/pkg/sql/show_zone_config.go +++ b/pkg/sql/show_zone_config.go @@ -383,7 +383,7 @@ func ascendZoneSpecifier( if actualID == keys.RootNamespaceID { // We had to traverse to the top of the hierarchy, so we're showing the // default zone config. - zs.NamedZone = zonepb.DefaultZoneName + zs.NamedZone = tree.UnrestrictedName(zonepb.DefaultZoneName) zs.Database = "" zs.TableOrIndex = tree.TableIndexName{} // Since the default zone has no partition, we can erase the diff --git a/pkg/sql/zone_config.go b/pkg/sql/zone_config.go index e0d2dd1c24d8..9688eb8aa7ff 100644 --- a/pkg/sql/zone_config.go +++ b/pkg/sql/zone_config.go @@ -258,6 +258,96 @@ func GetZoneConfigInTxn( return zoneID, zone, subzone, nil } +// GetHydratedZoneConfigForNamedZone returns a zone config for the given named +// zone. Any missing fields are filled through the RANGE DEFAULT zone config. +func GetHydratedZoneConfigForNamedZone( + ctx context.Context, txn *kv.Txn, codec keys.SQLCodec, zoneName zonepb.NamedZone, +) (*zonepb.ZoneConfig, error) { + getKey := func(key roachpb.Key) (*roachpb.Value, error) { + kv, err := txn.Get(ctx, key) + if err != nil { + return nil, err + } + return kv.Value, nil + } + id, found := zonepb.NamedZones[zoneName] + if !found { + return nil, errors.AssertionFailedf("id %d does not belong to a named zone", id) + } + zoneID, zone, _, _, err := getZoneConfig( + codec, descpb.ID(id), getKey, false /* getInheritedDefault */, false, /* mayBeTable */ + ) + if err := completeZoneConfig(zone, codec, zoneID, getKey); err != nil { + return nil, err + } + return zone, err +} + +// GetHydratedZoneConfigForTable returns a fully hydrated zone config for a +// given table ID. +func GetHydratedZoneConfigForTable( + ctx context.Context, txn *kv.Txn, codec keys.SQLCodec, id descpb.ID, +) (*zonepb.ZoneConfig, error) { + getKey := func(key roachpb.Key) (*roachpb.Value, error) { + kv, err := txn.Get(ctx, key) + if err != nil { + return nil, err + } + return kv.Value, nil + } + // TODO(arul): Teach `getZoneConfig` to use a descriptor collection instead of + // using this getKey function above to do descriptor lookups. + zoneID, zone, _, placeholder, err := getZoneConfig( + codec, id, getKey, false /* getInheritedDefault */, true, /* mayBeTable */ + ) + if err != nil { + return nil, err + } + if err := completeZoneConfig(zone, codec, zoneID, getKey); err != nil { + return nil, err + } + + // We've completely hydrated the zone config now. The only thing left to do + // is to do is hydrate the subzones, if applicable. + + // A placeholder config exists only to store subzones, so we copy over that + // information on the zone config. + if placeholder != nil { + // A placeholder config only exists for tables. Furthermore, if it exists, + // then the zone config (`zone`) above must belong to an object further up + // in the inheritance chain (such as the database or DEFAULT RANGE). As the + // subzones field is only defined if the zone config applies to a table, it + // follows that `zone` must not have any Subzones set on it. + if len(zone.Subzones) != 0 { + return nil, errors.AssertionFailedf("placeholder %v exists in conjunction with subzones on zone config %v", *zone, *placeholder) + } + zone.Subzones = placeholder.Subzones + zone.SubzoneSpans = placeholder.SubzoneSpans + } + + for i, subzone := range zone.Subzones { + // Check if a zone configuration exists for the index this subzone applies + // to by passing in a an empty partition below. + indexSubzone := zone.GetSubzone(subzone.IndexID, "" /* partition */) + // Partitions, in terms of the inheritance hierarchy, first inherit from the + // zone configuration fields on their parent index (if such a zone + // configuration exists). + // NB: If the subzone we're dealing with belongs to an index and not a + // partition, then the call below is a no-op. + if indexSubzone != nil { + zone.Subzones[i].Config.InheritFromParent(&indexSubzone.Config) + } + // After inheriting from the index's zone configuration, any fields that are + // left empty must be filled in from the table's zone configuration. Note + // that the table's zone configuration was fully hydrated above and + // inheriting from it will result in the subzone config being fully hydrated + // as well. + zone.Subzones[i].Config.InheritFromParent(zone) + } + + return zone, nil +} + func zoneSpecifierNotFoundError(zs tree.ZoneSpecifier) error { if zs.NamedZone != "" { return pgerror.Newf( diff --git a/pkg/testutils/serverutils/test_server_shim.go b/pkg/testutils/serverutils/test_server_shim.go index 9c64689a3968..8aca2cb17a41 100644 --- a/pkg/testutils/serverutils/test_server_shim.go +++ b/pkg/testutils/serverutils/test_server_shim.go @@ -118,6 +118,10 @@ type TestServerInterface interface { // interface{}. SpanConfigAccessor() interface{} + // SpanConfigSQLTranslator returns the underlying spanconfig.SQLTranslator as + // an interface{}. + SpanConfigSQLTranslator() interface{} + // SQLServer returns the *sql.Server as an interface{}. SQLServer() interface{}