From db5f5db46a87b3de4409d61a683e403e6b271a78 Mon Sep 17 00:00:00 2001 From: Rebecca Taft Date: Tue, 5 Feb 2019 14:08:12 -0500 Subject: [PATCH 01/10] importccl: fix flaky test TestImportCSVStmt TestImportCSVStmt tests that IMPORT jobs appear in a certain order in the system.jobs table. Automatic statistics were causing this test to be flaky since CreateStats jobs were present in the jobs table as well, in an unpredictable order. This commit fixes the problem by only selecting IMPORT jobs from the jobs table. Fixes #34568 Release note: None --- pkg/ccl/backupccl/backup_test.go | 7 ++---- pkg/ccl/importccl/import_stmt_test.go | 23 +++++++++-------- pkg/testutils/jobutils/jobs_verification.go | 28 +++++++-------------- 3 files changed, 23 insertions(+), 35 deletions(-) diff --git a/pkg/ccl/backupccl/backup_test.go b/pkg/ccl/backupccl/backup_test.go index ac92f3c49223..a2ecbc816521 100644 --- a/pkg/ccl/backupccl/backup_test.go +++ b/pkg/ccl/backupccl/backup_test.go @@ -435,9 +435,6 @@ func TestBackupRestoreSystemJobs(t *testing.T) { conn := sqlDB.DB.(*gosql.DB) defer cleanupFn() - // Get the number of existing jobs. - baseNumJobs := jobutils.GetSystemJobsCount(t, sqlDB) - sanitizedIncDir := localFoo + "/inc" incDir := sanitizedIncDir + "?secretCredentialsHere" @@ -462,7 +459,7 @@ func TestBackupRestoreSystemJobs(t *testing.T) { sqlDB.Exec(t, `SET DATABASE = data`) sqlDB.Exec(t, `BACKUP TABLE bank TO $1 INCREMENTAL FROM $2`, incDir, fullDir) - if err := jobutils.VerifySystemJob(t, sqlDB, baseNumJobs+1, jobspb.TypeBackup, jobs.StatusSucceeded, jobs.Record{ + if err := jobutils.VerifySystemJob(t, sqlDB, 1, jobspb.TypeBackup, jobs.StatusSucceeded, jobs.Record{ Username: security.RootUser, Description: fmt.Sprintf( `BACKUP TABLE bank TO '%s' INCREMENTAL FROM '%s'`, @@ -477,7 +474,7 @@ func TestBackupRestoreSystemJobs(t *testing.T) { } sqlDB.Exec(t, `RESTORE TABLE bank FROM $1, $2 WITH OPTIONS ('into_db'='restoredb')`, fullDir, incDir) - if err := jobutils.VerifySystemJob(t, sqlDB, baseNumJobs+2, jobspb.TypeRestore, jobs.StatusSucceeded, jobs.Record{ + if err := jobutils.VerifySystemJob(t, sqlDB, 0, jobspb.TypeRestore, jobs.StatusSucceeded, jobs.Record{ Username: security.RootUser, Description: fmt.Sprintf( `RESTORE TABLE bank FROM '%s', '%s' WITH into_db = 'restoredb'`, diff --git a/pkg/ccl/importccl/import_stmt_test.go b/pkg/ccl/importccl/import_stmt_test.go index 47b1f214115f..20a8311d50b8 100644 --- a/pkg/ccl/importccl/import_stmt_test.go +++ b/pkg/ccl/importccl/import_stmt_test.go @@ -867,13 +867,18 @@ func TestImportCSVStmt(t *testing.T) { if testing.Short() { t.Skip("short") } - t.Skip(`#34568`) - const ( - nodes = 3 - numFiles = nodes + 2 - rowsPerFile = 1000 - ) + const nodes = 3 + + numFiles := nodes + 2 + rowsPerFile := 1000 + if util.RaceEnabled { + // This test takes a while with the race detector, so reduce the number of + // files and rows per file in an attempt to speed it up. + numFiles = nodes + rowsPerFile = 16 + } + ctx := context.Background() dir, cleanup := testutils.TempDir(t) defer cleanup() @@ -897,9 +902,6 @@ func TestImportCSVStmt(t *testing.T) { t.Fatal(err) } - // Get the number of existing jobs. - baseNumJobs := jobutils.GetSystemJobsCount(t, sqlDB) - if err := ioutil.WriteFile(filepath.Join(dir, "empty.csv"), nil, 0666); err != nil { t.Fatal(err) } @@ -1174,7 +1176,7 @@ func TestImportCSVStmt(t *testing.T) { } jobPrefix += `t (a INT8 PRIMARY KEY, b STRING, INDEX (b), INDEX (a, b)) CSV DATA (%s)` - if err := jobutils.VerifySystemJob(t, sqlDB, baseNumJobs+testNum, jobspb.TypeImport, jobs.StatusSucceeded, jobs.Record{ + if err := jobutils.VerifySystemJob(t, sqlDB, testNum, jobspb.TypeImport, jobs.StatusSucceeded, jobs.Record{ Username: security.RootUser, Description: fmt.Sprintf(jobPrefix+tc.jobOpts, strings.Join(tc.files, ", ")), }); err != nil { @@ -1188,7 +1190,6 @@ func TestImportCSVStmt(t *testing.T) { t, "does not exist", `SELECT count(*) FROM t`, ) - testNum++ sqlDB.QueryRow( t, `RESTORE csv.* FROM $1 WITH into_db = $2`, backupPath, intodb, ).Scan( diff --git a/pkg/testutils/jobutils/jobs_verification.go b/pkg/testutils/jobutils/jobs_verification.go index e5292aaf6a06..0f26fd9ff0d2 100644 --- a/pkg/testutils/jobutils/jobs_verification.go +++ b/pkg/testutils/jobutils/jobs_verification.go @@ -118,25 +118,17 @@ func BulkOpResponseFilter(allowProgressIota *chan struct{}) storagebase.ReplicaR } } -// GetSystemJobsCount queries the number of entries in the jobs table. -func GetSystemJobsCount(t testing.TB, db *sqlutils.SQLRunner) int { - var jobCount int - db.QueryRow(t, `SELECT count(*) FROM crdb_internal.jobs`).Scan(&jobCount) - return jobCount -} - func verifySystemJob( t testing.TB, db *sqlutils.SQLRunner, offset int, - expectedType jobspb.Type, + filterType jobspb.Type, expectedStatus string, expectedRunningStatus string, expected jobs.Record, ) error { var actual jobs.Record var rawDescriptorIDs pq.Int64Array - var actualType string var statusString string var runningStatus gosql.NullString var runningStatusString string @@ -144,11 +136,12 @@ func verifySystemJob( // because job-generating SQL queries (e.g. BACKUP) do not currently return // the job ID. db.QueryRow(t, ` - SELECT job_type, description, user_name, descriptor_ids, status, running_status - FROM crdb_internal.jobs ORDER BY created LIMIT 1 OFFSET $1`, + SELECT description, user_name, descriptor_ids, status, running_status + FROM crdb_internal.jobs WHERE job_type = $1 ORDER BY created LIMIT 1 OFFSET $2`, + filterType.String(), offset, ).Scan( - &actualType, &actual.Description, &actual.Username, &rawDescriptorIDs, + &actual.Description, &actual.Username, &rawDescriptorIDs, &statusString, &runningStatus, ) if runningStatus.Valid { @@ -173,9 +166,6 @@ func verifySystemJob( return errors.Errorf("job %d: expected running status %v, got %v", offset, expectedRunningStatus, runningStatusString) } - if e, a := expectedType.String(), actualType; e != a { - return errors.Errorf("job %d: expected type %v, got type %v", offset, e, a) - } return nil } @@ -186,11 +176,11 @@ func VerifyRunningSystemJob( t testing.TB, db *sqlutils.SQLRunner, offset int, - expectedType jobspb.Type, + filterType jobspb.Type, expectedRunningStatus jobs.RunningStatus, expected jobs.Record, ) error { - return verifySystemJob(t, db, offset, expectedType, "running", string(expectedRunningStatus), expected) + return verifySystemJob(t, db, offset, filterType, "running", string(expectedRunningStatus), expected) } // VerifySystemJob checks that job records are created as expected. @@ -198,11 +188,11 @@ func VerifySystemJob( t testing.TB, db *sqlutils.SQLRunner, offset int, - expectedType jobspb.Type, + filterType jobspb.Type, expectedStatus jobs.Status, expected jobs.Record, ) error { - return verifySystemJob(t, db, offset, expectedType, string(expectedStatus), "", expected) + return verifySystemJob(t, db, offset, filterType, string(expectedStatus), "", expected) } // GetJobID gets a particular job's ID. From 471df55a8538913ab326377f1f186fa27eb45e2c Mon Sep 17 00:00:00 2001 From: Jordan Lewis Date: Sat, 9 Feb 2019 18:18:57 -0500 Subject: [PATCH 02/10] distsqlplan: fix error in union planning Previously, if 2 inputs to a UNION ALL had identical post processing except for renders, further post processing on top of that union all could invalidate the plan and cause errors or crashes. Release note (bug fix): fix a planning crash during UNION ALL operations that had projections, filters or renders directly on top of the UNION ALL in some cases. --- pkg/sql/distsqlplan/physical_plan.go | 11 +++++++++-- pkg/sql/logictest/testdata/logic_test/sqlsmith | 13 +++++++++++++ pkg/sql/logictest/testdata/logic_test/union | 5 +++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pkg/sql/distsqlplan/physical_plan.go b/pkg/sql/distsqlplan/physical_plan.go index 4bb0ddf48905..1e8ec7e6072e 100644 --- a/pkg/sql/distsqlplan/physical_plan.go +++ b/pkg/sql/distsqlplan/physical_plan.go @@ -283,8 +283,10 @@ func (p *PhysicalPlan) CheckLastStagePost() error { // verify this assumption. for i := 1; i < len(p.ResultRouters); i++ { pi := &p.Processors[p.ResultRouters[i]].Spec.Post - if pi.Filter != post.Filter || pi.Projection != post.Projection || - len(pi.OutputColumns) != len(post.OutputColumns) { + if pi.Filter != post.Filter || + pi.Projection != post.Projection || + len(pi.OutputColumns) != len(post.OutputColumns) || + len(pi.RenderExprs) != len(post.RenderExprs) { return errors.Errorf("inconsistent post-processing: %v vs %v", post, pi) } for j, col := range pi.OutputColumns { @@ -292,6 +294,11 @@ func (p *PhysicalPlan) CheckLastStagePost() error { return errors.Errorf("inconsistent post-processing: %v vs %v", post, pi) } } + for j, col := range pi.RenderExprs { + if col != post.RenderExprs[j] { + return errors.Errorf("inconsistent post-processing: %v vs %v", post, pi) + } + } } return nil diff --git a/pkg/sql/logictest/testdata/logic_test/sqlsmith b/pkg/sql/logictest/testdata/logic_test/sqlsmith index 1da0a1b412c9..870fd97f29f5 100644 --- a/pkg/sql/logictest/testdata/logic_test/sqlsmith +++ b/pkg/sql/logictest/testdata/logic_test/sqlsmith @@ -16,3 +16,16 @@ CREATE TABLE a (a INT PRIMARY KEY); statement ok SELECT true FROM (SELECT ref_1.a AS c0 FROM crdb_internal.cluster_queries AS ref_0 JOIN a AS ref_1 ON (ref_0.node_id = ref_1.a) WHERE (SELECT a from a limit 1 offset 1) is null); + +# Regression: #34437 (union all could produce panic in distsql planning) + +statement ok +CREATE TABLE table8 (col1 TIME, col2 BYTES, col4 OID, col6 NAME, col9 TIMESTAMP, PRIMARY KEY (col1)); + +statement ok +CREATE TABLE table5 (col0 TIME NULL, col1 OID, col3 INET, PRIMARY KEY (col1 ASC)); + +statement ok +INSERT INTO table8 (col1, col2, col4, col6) +VALUES ('19:06:18.321589', NULL, NULL, NULL) +UNION ALL (SELECT NULL, NULL, NULL, NULL FROM table5 AS tab_8); diff --git a/pkg/sql/logictest/testdata/logic_test/union b/pkg/sql/logictest/testdata/logic_test/union index 1dee30ed870c..97828a669ff3 100644 --- a/pkg/sql/logictest/testdata/logic_test/union +++ b/pkg/sql/logictest/testdata/logic_test/union @@ -246,6 +246,11 @@ query I SELECT a FROM a WHERE a > 2 UNION ALL (SELECT a FROM c WHERE b > 2) LIMIT 1; ---- +query III +select *,1 from (values(1,2) union all select 2,2 from c); +---- +1 2 1 + statement ok INSERT INTO a VALUES (1) From 828b925ceed4ebcecdd09c34d49c12ce7cc924c9 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Mon, 11 Feb 2019 11:23:20 -0500 Subject: [PATCH 03/10] opt: add stats to tpch xform test Since we have stats by default now, this should be the default testing mechanism. I left in tpch-no-stats since we also have that for tpcc, just for safety. Release note: None --- pkg/sql/opt/xform/testdata/external/tpch | 890 +++--- .../opt/xform/testdata/external/tpch-no-stats | 2737 +++++++++++++++++ 2 files changed, 3266 insertions(+), 361 deletions(-) create mode 100644 pkg/sql/opt/xform/testdata/external/tpch-no-stats diff --git a/pkg/sql/opt/xform/testdata/external/tpch b/pkg/sql/opt/xform/testdata/external/tpch index ce74518b703e..f32cfa915300 100644 --- a/pkg/sql/opt/xform/testdata/external/tpch +++ b/pkg/sql/opt/xform/testdata/external/tpch @@ -13,6 +13,17 @@ TABLE region └── INDEX primary └── r_regionkey int not null +exec-ddl +ALTER TABLE region INJECT STATISTICS '[ + { + "columns": ["r_regionkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 5, + "distinct_count": 5 + } +]' +---- + exec-ddl CREATE TABLE public.nation ( @@ -36,6 +47,23 @@ TABLE nation │ └── n_nationkey int not null └── FOREIGN KEY (n_regionkey) REFERENCES t.public.region (r_regionkey) +exec-ddl +ALTER TABLE nation INJECT STATISTICS '[ + { + "columns": ["n_nationkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 25, + "distinct_count": 25 + }, + { + "columns": ["n_regionkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 25, + "distinct_count": 5 + } +]' +---- + exec-ddl CREATE TABLE public.supplier ( @@ -65,6 +93,23 @@ TABLE supplier │ └── s_suppkey int not null └── FOREIGN KEY (s_nationkey) REFERENCES t.public.nation (n_nationkey) +exec-ddl +ALTER TABLE supplier INJECT STATISTICS '[ + { + "columns": ["s_suppkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 10000, + "distinct_count": 10000 + }, + { + "columns": ["s_nationkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 10000, + "distinct_count": 25 + } +]' +---- + exec-ddl CREATE TABLE public.part ( @@ -92,6 +137,17 @@ TABLE part └── INDEX primary └── p_partkey int not null +exec-ddl +ALTER TABLE part INJECT STATISTICS '[ + { + "columns": ["p_partkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 200000, + "distinct_count": 200000 + } +]' +---- + exec-ddl CREATE TABLE public.partsupp ( @@ -121,6 +177,23 @@ TABLE partsupp ├── FOREIGN KEY (ps_partkey) REFERENCES t.public.part (p_partkey) └── FOREIGN KEY (ps_suppkey) REFERENCES t.public.supplier (s_suppkey) +exec-ddl +ALTER TABLE partsupp INJECT STATISTICS '[ + { + "columns": ["ps_partkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 800000, + "distinct_count": 200000 + }, + { + "columns": ["ps_suppkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 800000, + "distinct_count": 10000 + } +]' +---- + exec-ddl CREATE TABLE public.customer ( @@ -152,6 +225,23 @@ TABLE customer │ └── c_custkey int not null └── FOREIGN KEY (c_nationkey) REFERENCES t.public.nation (n_nationkey) +exec-ddl +ALTER TABLE customer INJECT STATISTICS '[ + { + "columns": ["c_custkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 150000, + "distinct_count": 150000 + }, + { + "columns": ["c_nationkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 150000, + "distinct_count": 25 + } +]' +---- + exec-ddl CREATE TABLE public.orders ( @@ -189,6 +279,29 @@ TABLE orders │ └── o_orderkey int not null └── FOREIGN KEY (o_custkey) REFERENCES t.public.customer (c_custkey) +exec-ddl +ALTER TABLE orders INJECT STATISTICS '[ + { + "columns": ["o_orderkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 1500000, + "distinct_count": 1500000 + }, + { + "columns": ["o_custkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 1500000, + "distinct_count": 10000 + }, + { + "columns": ["o_orderdate"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 1500000, + "distinct_count": 2500 + } +]' +---- + exec-ddl CREATE TABLE public.lineitem ( @@ -281,6 +394,48 @@ TABLE lineitem ├── FOREIGN KEY (l_suppkey) REFERENCES t.public.supplier (s_suppkey) └── FOREIGN KEY (l_partkey, l_suppkey) REFERENCES t.public.partsupp (ps_partkey, ps_suppkey) +exec-ddl +ALTER TABLE lineitem INJECT STATISTICS '[ + { + "columns": ["l_orderkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 6000000, + "distinct_count": 1500000 + }, + { + "columns": ["l_partkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 6000000, + "distinct_count": 200000 + }, + { + "columns": ["l_suppkey"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 6000000, + "distinct_count": 10000 + }, + { + "columns": ["l_shipdate"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 6000000, + "distinct_count": 2500 + }, + { + "columns": ["l_commitdate"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 6000000, + "distinct_count": 2500 + }, + { + "columns": ["l_receiptdate"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 6000000, + "distinct_count": 2500 + } +]' +---- + + # -------------------------------------------------- # Q1 # Pricing Summary Report @@ -458,32 +613,36 @@ project │ │ │ │ │ ├── columns: ps_partkey:29(int!null) ps_suppkey:30(int!null) ps_supplycost:32(float!null) │ │ │ │ │ ├── key: (29,30) │ │ │ │ │ └── fd: (29,30)-->(32) - │ │ │ │ ├── inner-join + │ │ │ │ ├── inner-join (lookup supplier@s_nk) │ │ │ │ │ ├── columns: s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ ├── key columns: [41] = [37] │ │ │ │ │ ├── key: (34) │ │ │ │ │ ├── fd: ()-->(46), (34)-->(37), (41)-->(43), (43)==(45), (45)==(43), (37)==(41), (41)==(37) - │ │ │ │ │ ├── scan supplier@s_nk - │ │ │ │ │ │ ├── columns: s_suppkey:34(int!null) s_nationkey:37(int!null) - │ │ │ │ │ │ ├── key: (34) - │ │ │ │ │ │ └── fd: (34)-->(37) - │ │ │ │ │ ├── inner-join (lookup nation@n_rk) + │ │ │ │ │ ├── inner-join (merge) │ │ │ │ │ │ ├── columns: n_nationkey:41(int!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) - │ │ │ │ │ │ ├── key columns: [45] = [43] + │ │ │ │ │ │ ├── left ordering: +43 + │ │ │ │ │ │ ├── right ordering: +45 │ │ │ │ │ │ ├── key: (41) │ │ │ │ │ │ ├── fd: ()-->(46), (41)-->(43), (43)==(45), (45)==(43) + │ │ │ │ │ │ ├── scan nation@n_rk + │ │ │ │ │ │ │ ├── columns: n_nationkey:41(int!null) n_regionkey:43(int!null) + │ │ │ │ │ │ │ ├── key: (41) + │ │ │ │ │ │ │ ├── fd: (41)-->(43) + │ │ │ │ │ │ │ └── ordering: +43 │ │ │ │ │ │ ├── select │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) │ │ │ │ │ │ │ ├── key: (45) │ │ │ │ │ │ │ ├── fd: ()-->(46) + │ │ │ │ │ │ │ ├── ordering: +45 opt(46) [actual: +45] │ │ │ │ │ │ │ ├── scan region │ │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) │ │ │ │ │ │ │ │ ├── key: (45) - │ │ │ │ │ │ │ │ └── fd: (45)-->(46) + │ │ │ │ │ │ │ │ ├── fd: (45)-->(46) + │ │ │ │ │ │ │ │ └── ordering: +45 opt(46) [actual: +45] │ │ │ │ │ │ │ └── filters │ │ │ │ │ │ │ └── r_name = 'EUROPE' [type=bool, outer=(46), constraints=(/46: [/'EUROPE' - /'EUROPE']; tight), fd=()-->(46)] │ │ │ │ │ │ └── filters (true) - │ │ │ │ │ └── filters - │ │ │ │ │ └── s_nationkey = n_nationkey [type=bool, outer=(37,41), constraints=(/37: (/NULL - ]; /41: (/NULL - ]), fd=(37)==(41), (41)==(37)] + │ │ │ │ │ └── filters (true) │ │ │ │ └── filters │ │ │ │ └── s_suppkey = ps_suppkey [type=bool, outer=(30,34), constraints=(/30: (/NULL - ]; /34: (/NULL - ]), fd=(30)==(34), (34)==(30)] │ │ │ ├── inner-join @@ -502,28 +661,26 @@ project │ │ │ │ │ │ │ ├── columns: ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) │ │ │ │ │ │ │ ├── key: (17,18) │ │ │ │ │ │ │ └── fd: (17,18)-->(20) - │ │ │ │ │ │ ├── inner-join (lookup nation) + │ │ │ │ │ │ ├── inner-join │ │ │ │ │ │ │ ├── columns: n_nationkey:22(int!null) n_name:23(string!null) n_regionkey:24(int!null) r_regionkey:26(int!null) r_name:27(string!null) - │ │ │ │ │ │ │ ├── key columns: [22] = [22] │ │ │ │ │ │ │ ├── key: (22) │ │ │ │ │ │ │ ├── fd: ()-->(27), (22)-->(23,24), (24)==(26), (26)==(24) - │ │ │ │ │ │ │ ├── inner-join (lookup nation@n_rk) - │ │ │ │ │ │ │ │ ├── columns: n_nationkey:22(int!null) n_regionkey:24(int!null) r_regionkey:26(int!null) r_name:27(string!null) - │ │ │ │ │ │ │ │ ├── key columns: [26] = [24] + │ │ │ │ │ │ │ ├── scan nation + │ │ │ │ │ │ │ │ ├── columns: n_nationkey:22(int!null) n_name:23(string!null) n_regionkey:24(int!null) │ │ │ │ │ │ │ │ ├── key: (22) - │ │ │ │ │ │ │ │ ├── fd: ()-->(27), (22)-->(24), (24)==(26), (26)==(24) - │ │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ │ └── fd: (22)-->(23,24) + │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ │ ├── columns: r_regionkey:26(int!null) r_name:27(string!null) + │ │ │ │ │ │ │ │ ├── key: (26) + │ │ │ │ │ │ │ │ ├── fd: ()-->(27) + │ │ │ │ │ │ │ │ ├── scan region │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:26(int!null) r_name:27(string!null) │ │ │ │ │ │ │ │ │ ├── key: (26) - │ │ │ │ │ │ │ │ │ ├── fd: ()-->(27) - │ │ │ │ │ │ │ │ │ ├── scan region - │ │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:26(int!null) r_name:27(string!null) - │ │ │ │ │ │ │ │ │ │ ├── key: (26) - │ │ │ │ │ │ │ │ │ │ └── fd: (26)-->(27) - │ │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ │ └── r_name = 'EUROPE' [type=bool, outer=(27), constraints=(/27: [/'EUROPE' - /'EUROPE']; tight), fd=()-->(27)] - │ │ │ │ │ │ │ │ └── filters (true) - │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ │ └── fd: (26)-->(27) + │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ └── r_name = 'EUROPE' [type=bool, outer=(27), constraints=(/27: [/'EUROPE' - /'EUROPE']; tight), fd=()-->(27)] + │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ └── n_regionkey = r_regionkey [type=bool, outer=(24,26), constraints=(/24: (/NULL - ]; /26: (/NULL - ]), fd=(24)==(26), (26)==(24)] │ │ │ │ │ │ └── filters (true) │ │ │ │ │ ├── scan supplier │ │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_name:11(string!null) s_address:12(string!null) s_nationkey:13(int!null) s_phone:14(string!null) s_acctbal:15(float!null) s_comment:16(string!null) @@ -631,32 +788,22 @@ limit │ │ ├── inner-join │ │ │ ├── columns: c_custkey:1(int!null) c_mktsegment:7(string!null) o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) o_shippriority:16(int!null) l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_shipdate:28(date!null) │ │ │ ├── fd: ()-->(7), (9)-->(10,13,16), (9)==(18), (18)==(9), (1)==(10), (10)==(1) - │ │ │ ├── inner-join (merge) + │ │ │ ├── inner-join (lookup lineitem) │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) o_shippriority:16(int!null) l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_shipdate:28(date!null) - │ │ │ │ ├── left ordering: +9 - │ │ │ │ ├── right ordering: +18 + │ │ │ │ ├── key columns: [9] = [18] │ │ │ │ ├── fd: (9)-->(10,13,16), (9)==(18), (18)==(9) │ │ │ │ ├── select │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) o_shippriority:16(int!null) │ │ │ │ │ ├── key: (9) │ │ │ │ │ ├── fd: (9)-->(10,13,16) - │ │ │ │ │ ├── ordering: +9 │ │ │ │ │ ├── scan orders │ │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) o_shippriority:16(int!null) │ │ │ │ │ │ ├── key: (9) - │ │ │ │ │ │ ├── fd: (9)-->(10,13,16) - │ │ │ │ │ │ └── ordering: +9 + │ │ │ │ │ │ └── fd: (9)-->(10,13,16) │ │ │ │ │ └── filters │ │ │ │ │ └── o_orderdate < '1995-03-15' [type=bool, outer=(13), constraints=(/13: (/NULL - /'1995-03-14']; tight)] - │ │ │ │ ├── select - │ │ │ │ │ ├── columns: l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_shipdate:28(date!null) - │ │ │ │ │ ├── ordering: +18 - │ │ │ │ │ ├── scan lineitem - │ │ │ │ │ │ ├── columns: l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_shipdate:28(date!null) - │ │ │ │ │ │ └── ordering: +18 - │ │ │ │ │ └── filters - │ │ │ │ │ └── l_shipdate > '1995-03-15' [type=bool, outer=(28), constraints=(/28: [/'1995-03-16' - ]; tight)] - │ │ │ │ └── filters (true) + │ │ │ │ └── filters + │ │ │ │ └── l_shipdate > '1995-03-15' [type=bool, outer=(28), constraints=(/28: [/'1995-03-16' - ]; tight)] │ │ │ ├── select │ │ │ │ ├── columns: c_custkey:1(int!null) c_mktsegment:7(string!null) │ │ │ │ ├── key: (1) @@ -814,38 +961,32 @@ sort │ │ │ │ ├── fd: ()-->(46), (34)-->(37), (41)-->(42,43), (43)==(45), (45)==(43), (37)==(41), (41)==(37), (20)==(34), (34)==(20) │ │ │ │ ├── scan lineitem │ │ │ │ │ └── columns: l_orderkey:18(int!null) l_suppkey:20(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) - │ │ │ │ ├── inner-join + │ │ │ │ ├── inner-join (lookup supplier@s_nk) │ │ │ │ │ ├── columns: s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_name:42(string!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ ├── key columns: [41] = [37] │ │ │ │ │ ├── key: (34) │ │ │ │ │ ├── fd: ()-->(46), (34)-->(37), (41)-->(42,43), (43)==(45), (45)==(43), (37)==(41), (41)==(37) - │ │ │ │ │ ├── scan supplier@s_nk - │ │ │ │ │ │ ├── columns: s_suppkey:34(int!null) s_nationkey:37(int!null) - │ │ │ │ │ │ ├── key: (34) - │ │ │ │ │ │ └── fd: (34)-->(37) - │ │ │ │ │ ├── inner-join (lookup nation) + │ │ │ │ │ ├── inner-join │ │ │ │ │ │ ├── columns: n_nationkey:41(int!null) n_name:42(string!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) - │ │ │ │ │ │ ├── key columns: [41] = [41] │ │ │ │ │ │ ├── key: (41) │ │ │ │ │ │ ├── fd: ()-->(46), (41)-->(42,43), (43)==(45), (45)==(43) - │ │ │ │ │ │ ├── inner-join (lookup nation@n_rk) - │ │ │ │ │ │ │ ├── columns: n_nationkey:41(int!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) - │ │ │ │ │ │ │ ├── key columns: [45] = [43] + │ │ │ │ │ │ ├── scan nation + │ │ │ │ │ │ │ ├── columns: n_nationkey:41(int!null) n_name:42(string!null) n_regionkey:43(int!null) │ │ │ │ │ │ │ ├── key: (41) - │ │ │ │ │ │ │ ├── fd: ()-->(46), (41)-->(43), (43)==(45), (45)==(43) - │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ └── fd: (41)-->(42,43) + │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ │ │ ├── key: (45) + │ │ │ │ │ │ │ ├── fd: ()-->(46) + │ │ │ │ │ │ │ ├── scan region │ │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) │ │ │ │ │ │ │ │ ├── key: (45) - │ │ │ │ │ │ │ │ ├── fd: ()-->(46) - │ │ │ │ │ │ │ │ ├── scan region - │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) - │ │ │ │ │ │ │ │ │ ├── key: (45) - │ │ │ │ │ │ │ │ │ └── fd: (45)-->(46) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ └── r_name = 'ASIA' [type=bool, outer=(46), constraints=(/46: [/'ASIA' - /'ASIA']; tight), fd=()-->(46)] - │ │ │ │ │ │ │ └── filters (true) - │ │ │ │ │ │ └── filters (true) - │ │ │ │ │ └── filters - │ │ │ │ │ └── s_nationkey = n_nationkey [type=bool, outer=(37,41), constraints=(/37: (/NULL - ]; /41: (/NULL - ]), fd=(37)==(41), (41)==(37)] + │ │ │ │ │ │ │ │ └── fd: (45)-->(46) + │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ └── r_name = 'ASIA' [type=bool, outer=(46), constraints=(/46: [/'ASIA' - /'ASIA']; tight), fd=()-->(46)] + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── n_regionkey = r_regionkey [type=bool, outer=(43,45), constraints=(/43: (/NULL - ]; /45: (/NULL - ]), fd=(43)==(45), (45)==(43)] + │ │ │ │ │ └── filters (true) │ │ │ │ └── filters │ │ │ │ └── l_suppkey = s_suppkey [type=bool, outer=(20,34), constraints=(/20: (/NULL - ]; /34: (/NULL - ]), fd=(20)==(34), (34)==(20)] │ │ │ ├── index-join orders @@ -979,79 +1120,87 @@ ORDER BY cust_nation, l_year; ---- -group-by +sort ├── columns: supp_nation:42(string!null) cust_nation:46(string!null) l_year:49(int) revenue:51(float) - ├── grouping columns: n1.n_name:42(string!null) n2.n_name:46(string!null) l_year:49(int) ├── key: (42,46,49) ├── fd: (42,46,49)-->(51) ├── ordering: +42,+46,+49 - ├── sort - │ ├── columns: n1.n_name:42(string!null) n2.n_name:46(string!null) l_year:49(int) volume:50(float) - │ ├── ordering: +42,+46,+49 - │ └── project - │ ├── columns: l_year:49(int) volume:50(float) n1.n_name:42(string!null) n2.n_name:46(string!null) - │ ├── inner-join - │ │ ├── columns: s_suppkey:1(int!null) s_nationkey:4(int!null) l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) - │ │ ├── fd: (1)-->(4), (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25), (8)==(24), (24)==(8), (1)==(10), (10)==(1), (4)==(41), (41)==(4) - │ │ ├── inner-join - │ │ │ ├── columns: l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) - │ │ │ ├── fd: (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25), (8)==(24), (24)==(8) - │ │ │ ├── inner-join - │ │ │ │ ├── columns: o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) - │ │ │ │ ├── key: (24,41) - │ │ │ │ ├── fd: (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25) - │ │ │ │ ├── inner-join - │ │ │ │ │ ├── columns: c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) - │ │ │ │ │ ├── key: (33,41) - │ │ │ │ │ ├── fd: (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36) - │ │ │ │ │ ├── inner-join - │ │ │ │ │ │ ├── columns: n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) - │ │ │ │ │ │ ├── key: (41,45) - │ │ │ │ │ │ ├── fd: (41)-->(42), (45)-->(46) - │ │ │ │ │ │ ├── scan n1 - │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:41(int!null) n1.n_name:42(string!null) - │ │ │ │ │ │ │ ├── key: (41) - │ │ │ │ │ │ │ └── fd: (41)-->(42) - │ │ │ │ │ │ ├── scan n2 - │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:45(int!null) n2.n_name:46(string!null) - │ │ │ │ │ │ │ ├── key: (45) - │ │ │ │ │ │ │ └── fd: (45)-->(46) - │ │ │ │ │ │ └── filters - │ │ │ │ │ │ └── ((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')) [type=bool, outer=(42,46)] - │ │ │ │ │ ├── scan customer@c_nk - │ │ │ │ │ │ ├── columns: c_custkey:33(int!null) c_nationkey:36(int!null) - │ │ │ │ │ │ ├── key: (33) - │ │ │ │ │ │ └── fd: (33)-->(36) - │ │ │ │ │ └── filters - │ │ │ │ │ └── c_nationkey = n2.n_nationkey [type=bool, outer=(36,45), constraints=(/36: (/NULL - ]; /45: (/NULL - ]), fd=(36)==(45), (45)==(36)] - │ │ │ │ ├── scan orders@o_ck - │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_custkey:25(int!null) - │ │ │ │ │ ├── key: (24) - │ │ │ │ │ └── fd: (24)-->(25) - │ │ │ │ └── filters - │ │ │ │ └── c_custkey = o_custkey [type=bool, outer=(25,33), constraints=(/25: (/NULL - ]; /33: (/NULL - ]), fd=(25)==(33), (33)==(25)] - │ │ │ ├── index-join lineitem - │ │ │ │ ├── columns: l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) - │ │ │ │ └── scan lineitem@l_sd - │ │ │ │ ├── columns: l_orderkey:8(int!null) l_linenumber:11(int!null) l_shipdate:18(date!null) - │ │ │ │ ├── constraint: /18/8/11: [/'1995-01-01' - /'1996-12-31'] - │ │ │ │ ├── key: (8,11) - │ │ │ │ └── fd: (8,11)-->(18) - │ │ │ └── filters - │ │ │ └── o_orderkey = l_orderkey [type=bool, outer=(8,24), constraints=(/8: (/NULL - ]; /24: (/NULL - ]), fd=(8)==(24), (24)==(8)] - │ │ ├── scan supplier@s_nk - │ │ │ ├── columns: s_suppkey:1(int!null) s_nationkey:4(int!null) - │ │ │ ├── key: (1) - │ │ │ └── fd: (1)-->(4) - │ │ └── filters - │ │ ├── s_suppkey = l_suppkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] - │ │ └── s_nationkey = n1.n_nationkey [type=bool, outer=(4,41), constraints=(/4: (/NULL - ]; /41: (/NULL - ]), fd=(4)==(41), (41)==(4)] - │ └── projections - │ ├── extract('year', l_shipdate) [type=int, outer=(18)] - │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(13,14)] - └── aggregations - └── sum [type=float, outer=(50)] - └── variable: volume [type=float] + └── group-by + ├── columns: n1.n_name:42(string!null) n2.n_name:46(string!null) l_year:49(int) sum:51(float) + ├── grouping columns: n1.n_name:42(string!null) n2.n_name:46(string!null) l_year:49(int) + ├── key: (42,46,49) + ├── fd: (42,46,49)-->(51) + ├── project + │ ├── columns: l_year:49(int) volume:50(float) n1.n_name:42(string!null) n2.n_name:46(string!null) + │ ├── inner-join + │ │ ├── columns: s_suppkey:1(int!null) s_nationkey:4(int!null) l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ ├── fd: (1)-->(4), (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25), (8)==(24), (24)==(8), (1)==(10), (10)==(1), (4)==(41), (41)==(4) + │ │ ├── inner-join + │ │ │ ├── columns: l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ ├── fd: (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25), (8)==(24), (24)==(8) + │ │ │ ├── inner-join + │ │ │ │ ├── columns: o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ ├── key: (24,41) + │ │ │ │ ├── fd: (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25) + │ │ │ │ ├── scan orders@o_ck + │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_custkey:25(int!null) + │ │ │ │ │ ├── key: (24) + │ │ │ │ │ └── fd: (24)-->(25) + │ │ │ │ ├── inner-join (merge) + │ │ │ │ │ ├── columns: c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ │ ├── left ordering: +36 + │ │ │ │ │ ├── right ordering: +45 + │ │ │ │ │ ├── key: (33,41) + │ │ │ │ │ ├── fd: (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36) + │ │ │ │ │ ├── scan customer@c_nk + │ │ │ │ │ │ ├── columns: c_custkey:33(int!null) c_nationkey:36(int!null) + │ │ │ │ │ │ ├── key: (33) + │ │ │ │ │ │ ├── fd: (33)-->(36) + │ │ │ │ │ │ └── ordering: +36 + │ │ │ │ │ ├── sort + │ │ │ │ │ │ ├── columns: n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ │ │ ├── key: (41,45) + │ │ │ │ │ │ ├── fd: (41)-->(42), (45)-->(46) + │ │ │ │ │ │ ├── ordering: +45 + │ │ │ │ │ │ └── inner-join + │ │ │ │ │ │ ├── columns: n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ │ │ ├── key: (41,45) + │ │ │ │ │ │ ├── fd: (41)-->(42), (45)-->(46) + │ │ │ │ │ │ ├── scan n1 + │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:41(int!null) n1.n_name:42(string!null) + │ │ │ │ │ │ │ ├── key: (41) + │ │ │ │ │ │ │ └── fd: (41)-->(42) + │ │ │ │ │ │ ├── scan n2 + │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ │ │ │ ├── key: (45) + │ │ │ │ │ │ │ └── fd: (45)-->(46) + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── ((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')) [type=bool, outer=(42,46)] + │ │ │ │ │ └── filters (true) + │ │ │ │ └── filters + │ │ │ │ └── c_custkey = o_custkey [type=bool, outer=(25,33), constraints=(/25: (/NULL - ]; /33: (/NULL - ]), fd=(25)==(33), (33)==(25)] + │ │ │ ├── index-join lineitem + │ │ │ │ ├── columns: l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) + │ │ │ │ └── scan lineitem@l_sd + │ │ │ │ ├── columns: l_orderkey:8(int!null) l_linenumber:11(int!null) l_shipdate:18(date!null) + │ │ │ │ ├── constraint: /18/8/11: [/'1995-01-01' - /'1996-12-31'] + │ │ │ │ ├── key: (8,11) + │ │ │ │ └── fd: (8,11)-->(18) + │ │ │ └── filters + │ │ │ └── o_orderkey = l_orderkey [type=bool, outer=(8,24), constraints=(/8: (/NULL - ]; /24: (/NULL - ]), fd=(8)==(24), (24)==(8)] + │ │ ├── scan supplier@s_nk + │ │ │ ├── columns: s_suppkey:1(int!null) s_nationkey:4(int!null) + │ │ │ ├── key: (1) + │ │ │ └── fd: (1)-->(4) + │ │ └── filters + │ │ ├── s_suppkey = l_suppkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ └── s_nationkey = n1.n_nationkey [type=bool, outer=(4,41), constraints=(/4: (/NULL - ]; /41: (/NULL - ]), fd=(4)==(41), (41)==(4)] + │ └── projections + │ ├── extract('year', l_shipdate) [type=int, outer=(18)] + │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(13,14)] + └── aggregations + └── sum [type=float, outer=(50)] + └── variable: volume [type=float] # -------------------------------------------------- # Q8 @@ -1108,115 +1257,130 @@ GROUP BY ORDER BY o_year; ---- -sort +project ├── columns: o_year:61(int) mkt_share:66(float) ├── side-effects ├── key: (61) ├── fd: (61)-->(66) ├── ordering: +61 - └── project - ├── columns: mkt_share:66(float) o_year:61(int) - ├── side-effects - ├── key: (61) - ├── fd: (61)-->(66) - ├── group-by - │ ├── columns: o_year:61(int) sum:64(float) sum:65(float) - │ ├── grouping columns: o_year:61(int) - │ ├── key: (61) - │ ├── fd: (61)-->(64,65) - │ ├── project - │ │ ├── columns: column63:63(float) o_year:61(int) volume:62(float) - │ │ ├── project - │ │ │ ├── columns: o_year:61(int) volume:62(float) n2.n_name:55(string!null) - │ │ │ ├── inner-join (lookup part) - │ │ │ │ ├── columns: p_partkey:1(int!null) p_type:5(string!null) s_suppkey:10(int!null) s_nationkey:13(int!null) l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ ├── key columns: [18] = [1] - │ │ │ │ ├── fd: ()-->(5,59), (10)-->(13), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17), (10)==(19), (19)==(10), (13)==(54), (54)==(13), (1)==(18), (18)==(1) - │ │ │ │ ├── inner-join - │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_nationkey:13(int!null) l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ │ ├── fd: ()-->(59), (10)-->(13), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17), (10)==(19), (19)==(10), (13)==(54), (54)==(13) - │ │ │ │ │ ├── inner-join - │ │ │ │ │ │ ├── columns: l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ │ │ ├── fd: ()-->(59), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17) - │ │ │ │ │ │ ├── inner-join - │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ │ │ │ ├── key: (33,54) - │ │ │ │ │ │ │ ├── fd: ()-->(59), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34) - │ │ │ │ │ │ │ ├── inner-join - │ │ │ │ │ │ │ │ ├── columns: c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ │ │ │ │ ├── key: (42,54) - │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45) - │ │ │ │ │ │ │ │ ├── inner-join - │ │ │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ │ │ │ │ │ ├── key: (50,54) - │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52) - │ │ │ │ │ │ │ │ │ ├── inner-join - │ │ │ │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ │ │ │ │ │ │ ├── key: (54,58) - │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (54)-->(55) - │ │ │ │ │ │ │ │ │ │ ├── scan n2 - │ │ │ │ │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:54(int!null) n2.n_name:55(string!null) - │ │ │ │ │ │ │ │ │ │ │ ├── key: (54) - │ │ │ │ │ │ │ │ │ │ │ └── fd: (54)-->(55) - │ │ │ │ │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ │ │ │ │ │ │ │ ├── key: (58) - │ │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59) - │ │ │ │ │ │ │ │ │ │ │ ├── scan region - │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:58(int!null) r_name:59(string!null) - │ │ │ │ │ │ │ │ │ │ │ │ ├── key: (58) - │ │ │ │ │ │ │ │ │ │ │ │ └── fd: (58)-->(59) - │ │ │ │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ │ │ │ └── r_name = 'AMERICA' [type=bool, outer=(59), constraints=(/59: [/'AMERICA' - /'AMERICA']; tight), fd=()-->(59)] - │ │ │ │ │ │ │ │ │ │ └── filters (true) - │ │ │ │ │ │ │ │ │ ├── scan n1@n_rk - │ │ │ │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) - │ │ │ │ │ │ │ │ │ │ ├── key: (50) - │ │ │ │ │ │ │ │ │ │ └── fd: (50)-->(52) - │ │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ │ └── n1.n_regionkey = r_regionkey [type=bool, outer=(52,58), constraints=(/52: (/NULL - ]; /58: (/NULL - ]), fd=(52)==(58), (58)==(52)] - │ │ │ │ │ │ │ │ ├── scan customer@c_nk - │ │ │ │ │ │ │ │ │ ├── columns: c_custkey:42(int!null) c_nationkey:45(int!null) - │ │ │ │ │ │ │ │ │ ├── key: (42) - │ │ │ │ │ │ │ │ │ └── fd: (42)-->(45) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ └── c_nationkey = n1.n_nationkey [type=bool, outer=(45,50), constraints=(/45: (/NULL - ]; /50: (/NULL - ]), fd=(45)==(50), (50)==(45)] - │ │ │ │ │ │ │ ├── index-join orders - │ │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) - │ │ │ │ │ │ │ │ ├── key: (33) - │ │ │ │ │ │ │ │ ├── fd: (33)-->(34,37) - │ │ │ │ │ │ │ │ └── scan orders@o_od - │ │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_orderdate:37(date!null) - │ │ │ │ │ │ │ │ ├── constraint: /37/33: [/'1995-01-01' - /'1996-12-31'] - │ │ │ │ │ │ │ │ ├── key: (33) - │ │ │ │ │ │ │ │ └── fd: (33)-->(37) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ └── o_custkey = c_custkey [type=bool, outer=(34,42), constraints=(/34: (/NULL - ]; /42: (/NULL - ]), fd=(34)==(42), (42)==(34)] - │ │ │ │ │ │ ├── scan lineitem - │ │ │ │ │ │ │ └── columns: l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) - │ │ │ │ │ │ └── filters - │ │ │ │ │ │ └── l_orderkey = o_orderkey [type=bool, outer=(17,33), constraints=(/17: (/NULL - ]; /33: (/NULL - ]), fd=(17)==(33), (33)==(17)] - │ │ │ │ │ ├── scan supplier@s_nk - │ │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_nationkey:13(int!null) - │ │ │ │ │ │ ├── key: (10) - │ │ │ │ │ │ └── fd: (10)-->(13) - │ │ │ │ │ └── filters - │ │ │ │ │ ├── s_suppkey = l_suppkey [type=bool, outer=(10,19), constraints=(/10: (/NULL - ]; /19: (/NULL - ]), fd=(10)==(19), (19)==(10)] - │ │ │ │ │ └── s_nationkey = n2.n_nationkey [type=bool, outer=(13,54), constraints=(/13: (/NULL - ]; /54: (/NULL - ]), fd=(13)==(54), (54)==(13)] - │ │ │ │ └── filters - │ │ │ │ └── p_type = 'ECONOMY ANODIZED STEEL' [type=bool, outer=(5), constraints=(/5: [/'ECONOMY ANODIZED STEEL' - /'ECONOMY ANODIZED STEEL']; tight), fd=()-->(5)] - │ │ │ └── projections - │ │ │ ├── extract('year', o_orderdate) [type=int, outer=(37)] - │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(22,23)] - │ │ └── projections - │ │ └── CASE WHEN n2.n_name = 'BRAZIL' THEN volume ELSE 0.0 END [type=float, outer=(55,62)] - │ └── aggregations - │ ├── sum [type=float, outer=(63)] - │ │ └── variable: column63 [type=float] - │ └── sum [type=float, outer=(62)] - │ └── variable: volume [type=float] - └── projections - └── sum / sum [type=float, outer=(64,65), side-effects] + ├── group-by + │ ├── columns: o_year:61(int) sum:64(float) sum:65(float) + │ ├── grouping columns: o_year:61(int) + │ ├── key: (61) + │ ├── fd: (61)-->(64,65) + │ ├── ordering: +61 + │ ├── sort + │ │ ├── columns: o_year:61(int) volume:62(float) column63:63(float) + │ │ ├── ordering: +61 + │ │ └── project + │ │ ├── columns: column63:63(float) o_year:61(int) volume:62(float) + │ │ ├── project + │ │ │ ├── columns: o_year:61(int) volume:62(float) n2.n_name:55(string!null) + │ │ │ ├── inner-join + │ │ │ │ ├── columns: p_partkey:1(int!null) p_type:5(string!null) s_suppkey:10(int!null) s_nationkey:13(int!null) l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ ├── fd: ()-->(5,59), (10)-->(13), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17), (10)==(19), (19)==(10), (13)==(54), (54)==(13), (1)==(18), (18)==(1) + │ │ │ │ ├── inner-join + │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_nationkey:13(int!null) l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ ├── fd: ()-->(59), (10)-->(13), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17), (10)==(19), (19)==(10), (13)==(54), (54)==(13) + │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ ├── columns: l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ ├── fd: ()-->(59), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17) + │ │ │ │ │ │ ├── scan lineitem + │ │ │ │ │ │ │ └── columns: l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) + │ │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ ├── key: (33,54) + │ │ │ │ │ │ │ ├── fd: ()-->(59), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34) + │ │ │ │ │ │ │ ├── inner-join (merge) + │ │ │ │ │ │ │ │ ├── columns: c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ ├── left ordering: +45 + │ │ │ │ │ │ │ │ ├── right ordering: +50 + │ │ │ │ │ │ │ │ ├── key: (42,54) + │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45) + │ │ │ │ │ │ │ │ ├── scan customer@c_nk + │ │ │ │ │ │ │ │ │ ├── columns: c_custkey:42(int!null) c_nationkey:45(int!null) + │ │ │ │ │ │ │ │ │ ├── key: (42) + │ │ │ │ │ │ │ │ │ ├── fd: (42)-->(45) + │ │ │ │ │ │ │ │ │ └── ordering: +45 + │ │ │ │ │ │ │ │ ├── sort + │ │ │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ ├── key: (50,54) + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52) + │ │ │ │ │ │ │ │ │ ├── ordering: +50 opt(59) [actual: +50] + │ │ │ │ │ │ │ │ │ └── inner-join + │ │ │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ ├── key: (50,54) + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52) + │ │ │ │ │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ │ ├── key: (54,58) + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (54)-->(55) + │ │ │ │ │ │ │ │ │ │ ├── scan n2 + │ │ │ │ │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:54(int!null) n2.n_name:55(string!null) + │ │ │ │ │ │ │ │ │ │ │ ├── key: (54) + │ │ │ │ │ │ │ │ │ │ │ └── fd: (54)-->(55) + │ │ │ │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ │ │ ├── key: (58) + │ │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59) + │ │ │ │ │ │ │ │ │ │ │ ├── scan region + │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ │ │ │ ├── key: (58) + │ │ │ │ │ │ │ │ │ │ │ │ └── fd: (58)-->(59) + │ │ │ │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ │ │ │ └── r_name = 'AMERICA' [type=bool, outer=(59), constraints=(/59: [/'AMERICA' - /'AMERICA']; tight), fd=()-->(59)] + │ │ │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ │ ├── scan n1@n_rk + │ │ │ │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) + │ │ │ │ │ │ │ │ │ │ ├── key: (50) + │ │ │ │ │ │ │ │ │ │ └── fd: (50)-->(52) + │ │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ │ └── n1.n_regionkey = r_regionkey [type=bool, outer=(52,58), constraints=(/52: (/NULL - ]; /58: (/NULL - ]), fd=(52)==(58), (58)==(52)] + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ ├── index-join orders + │ │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) + │ │ │ │ │ │ │ │ ├── key: (33) + │ │ │ │ │ │ │ │ ├── fd: (33)-->(34,37) + │ │ │ │ │ │ │ │ └── scan orders@o_od + │ │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_orderdate:37(date!null) + │ │ │ │ │ │ │ │ ├── constraint: /37/33: [/'1995-01-01' - /'1996-12-31'] + │ │ │ │ │ │ │ │ ├── key: (33) + │ │ │ │ │ │ │ │ └── fd: (33)-->(37) + │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ └── o_custkey = c_custkey [type=bool, outer=(34,42), constraints=(/34: (/NULL - ]; /42: (/NULL - ]), fd=(34)==(42), (42)==(34)] + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── l_orderkey = o_orderkey [type=bool, outer=(17,33), constraints=(/17: (/NULL - ]; /33: (/NULL - ]), fd=(17)==(33), (33)==(17)] + │ │ │ │ │ ├── scan supplier@s_nk + │ │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_nationkey:13(int!null) + │ │ │ │ │ │ ├── key: (10) + │ │ │ │ │ │ └── fd: (10)-->(13) + │ │ │ │ │ └── filters + │ │ │ │ │ ├── s_suppkey = l_suppkey [type=bool, outer=(10,19), constraints=(/10: (/NULL - ]; /19: (/NULL - ]), fd=(10)==(19), (19)==(10)] + │ │ │ │ │ └── s_nationkey = n2.n_nationkey [type=bool, outer=(13,54), constraints=(/13: (/NULL - ]; /54: (/NULL - ]), fd=(13)==(54), (54)==(13)] + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: p_partkey:1(int!null) p_type:5(string!null) + │ │ │ │ │ ├── key: (1) + │ │ │ │ │ ├── fd: ()-->(5) + │ │ │ │ │ ├── scan part + │ │ │ │ │ │ ├── columns: p_partkey:1(int!null) p_type:5(string!null) + │ │ │ │ │ │ ├── key: (1) + │ │ │ │ │ │ └── fd: (1)-->(5) + │ │ │ │ │ └── filters + │ │ │ │ │ └── p_type = 'ECONOMY ANODIZED STEEL' [type=bool, outer=(5), constraints=(/5: [/'ECONOMY ANODIZED STEEL' - /'ECONOMY ANODIZED STEEL']; tight), fd=()-->(5)] + │ │ │ │ └── filters + │ │ │ │ └── p_partkey = l_partkey [type=bool, outer=(1,18), constraints=(/1: (/NULL - ]; /18: (/NULL - ]), fd=(1)==(18), (18)==(1)] + │ │ │ └── projections + │ │ │ ├── extract('year', o_orderdate) [type=int, outer=(37)] + │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(22,23)] + │ │ └── projections + │ │ └── CASE WHEN n2.n_name = 'BRAZIL' THEN volume ELSE 0.0 END [type=float, outer=(55,62)] + │ └── aggregations + │ ├── sum [type=float, outer=(63)] + │ │ └── variable: column63 [type=float] + │ └── sum [type=float, outer=(62)] + │ └── variable: volume [type=float] + └── projections + └── sum / sum [type=float, outer=(64,65), side-effects] # -------------------------------------------------- # Q9 @@ -1405,11 +1569,13 @@ limit │ ├── project │ │ ├── columns: column38:38(float) c_custkey:1(int!null) c_name:2(string!null) c_address:3(string!null) c_phone:5(string!null) c_acctbal:6(float!null) c_comment:8(string!null) n_name:35(string!null) │ │ ├── fd: (1)-->(2,3,5,6,8,35) - │ │ ├── inner-join + │ │ ├── inner-join (lookup customer) │ │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) c_address:3(string!null) c_nationkey:4(int!null) c_phone:5(string!null) c_acctbal:6(float!null) c_comment:8(string!null) o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_returnflag:26(string!null) n_nationkey:34(int!null) n_name:35(string!null) + │ │ │ ├── key columns: [10] = [1] │ │ │ ├── fd: ()-->(26), (1)-->(2-6,8), (9)-->(10,13), (34)-->(35), (9)==(18), (18)==(9), (1)==(10), (10)==(1), (4)==(34), (34)==(4) - │ │ │ ├── inner-join + │ │ │ ├── inner-join (lookup orders) │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_returnflag:26(string!null) n_nationkey:34(int!null) n_name:35(string!null) + │ │ │ │ ├── key columns: [18] = [9] │ │ │ │ ├── fd: ()-->(26), (9)-->(10,13), (34)-->(35), (9)==(18), (18)==(9) │ │ │ │ ├── inner-join │ │ │ │ │ ├── columns: l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_returnflag:26(string!null) n_nationkey:34(int!null) n_name:35(string!null) @@ -1426,23 +1592,10 @@ limit │ │ │ │ │ │ └── filters │ │ │ │ │ │ └── l_returnflag = 'R' [type=bool, outer=(26), constraints=(/26: [/'R' - /'R']; tight), fd=()-->(26)] │ │ │ │ │ └── filters (true) - │ │ │ │ ├── index-join orders - │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) - │ │ │ │ │ ├── key: (9) - │ │ │ │ │ ├── fd: (9)-->(10,13) - │ │ │ │ │ └── scan orders@o_od - │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_orderdate:13(date!null) - │ │ │ │ │ ├── constraint: /13/9: [/'1993-10-01' - /'1993-12-31'] - │ │ │ │ │ ├── key: (9) - │ │ │ │ │ └── fd: (9)-->(13) │ │ │ │ └── filters - │ │ │ │ └── l_orderkey = o_orderkey [type=bool, outer=(9,18), constraints=(/9: (/NULL - ]; /18: (/NULL - ]), fd=(9)==(18), (18)==(9)] - │ │ │ ├── scan customer - │ │ │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) c_address:3(string!null) c_nationkey:4(int!null) c_phone:5(string!null) c_acctbal:6(float!null) c_comment:8(string!null) - │ │ │ │ ├── key: (1) - │ │ │ │ └── fd: (1)-->(2-6,8) + │ │ │ │ ├── o_orderdate >= '1993-10-01' [type=bool, outer=(13), constraints=(/13: [/'1993-10-01' - ]; tight)] + │ │ │ │ └── o_orderdate < '1994-01-01' [type=bool, outer=(13), constraints=(/13: (/NULL - /'1993-12-31']; tight)] │ │ │ └── filters - │ │ │ ├── c_custkey = o_custkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] │ │ │ └── c_nationkey = n_nationkey [type=bool, outer=(4,34), constraints=(/4: (/NULL - ]; /34: (/NULL - ]), fd=(4)==(34), (34)==(4)] │ │ └── projections │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(23,24)] @@ -1727,13 +1880,10 @@ sort │ ├── grouping columns: c_custkey:1(int!null) │ ├── key: (1) │ ├── fd: (1)-->(18) - │ ├── left-join + │ ├── right-join │ │ ├── columns: c_custkey:1(int!null) o_orderkey:9(int) o_custkey:10(int) o_comment:17(string) │ │ ├── key: (1,9) │ │ ├── fd: (9)-->(10,17) - │ │ ├── scan customer@c_nk - │ │ │ ├── columns: c_custkey:1(int!null) - │ │ │ └── key: (1) │ │ ├── select │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_comment:17(string!null) │ │ │ ├── key: (9) @@ -1744,6 +1894,9 @@ sort │ │ │ │ └── fd: (9)-->(10,17) │ │ │ └── filters │ │ │ └── o_comment NOT LIKE '%special%requests%' [type=bool, outer=(17), constraints=(/17: (/NULL - ])] + │ │ ├── scan customer@c_nk + │ │ │ ├── columns: c_custkey:1(int!null) + │ │ │ └── key: (1) │ │ └── filters │ │ └── c_custkey = o_custkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] │ └── aggregations @@ -1791,9 +1944,8 @@ project │ ├── fd: ()-->(27,29) │ ├── project │ │ ├── columns: column26:26(float) column28:28(float) - │ │ ├── inner-join (lookup part) + │ │ ├── inner-join │ │ │ ├── columns: l_partkey:2(int!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipdate:11(date!null) p_partkey:17(int!null) p_type:21(string!null) - │ │ │ ├── key columns: [2] = [17] │ │ │ ├── fd: (17)-->(21), (2)==(17), (17)==(2) │ │ │ ├── index-join lineitem │ │ │ │ ├── columns: l_partkey:2(int!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipdate:11(date!null) @@ -1802,7 +1954,12 @@ project │ │ │ │ ├── constraint: /11/1/4: [/'1995-09-01' - /'1995-09-30'] │ │ │ │ ├── key: (1,4) │ │ │ │ └── fd: (1,4)-->(11) - │ │ │ └── filters (true) + │ │ │ ├── scan part + │ │ │ │ ├── columns: p_partkey:17(int!null) p_type:21(string!null) + │ │ │ │ ├── key: (17) + │ │ │ │ └── fd: (17)-->(21) + │ │ │ └── filters + │ │ │ └── l_partkey = p_partkey [type=bool, outer=(2,17), constraints=(/2: (/NULL - ]; /17: (/NULL - ]), fd=(2)==(17), (17)==(2)] │ │ └── projections │ │ ├── CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1.0 - l_discount) ELSE 0.0 END [type=float, outer=(6,7,21)] │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(6,7)] @@ -1862,75 +2019,83 @@ WHERE ORDER BY s_suppkey; ---- -sort +project ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_phone:5(string!null) total_revenue:25(float!null) ├── key: (1) ├── fd: (1)-->(2,3,5,25) ├── ordering: +1 - └── project - ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_phone:5(string!null) sum:25(float!null) - ├── key: (1) - ├── fd: (1)-->(2,3,5,25) - └── inner-join (lookup supplier) - ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_phone:5(string!null) l_suppkey:10(int!null) sum:25(float!null) - ├── key columns: [10] = [1] - ├── key: (10) - ├── fd: (1)-->(2,3,5), (10)-->(25), (1)==(10), (10)==(1) - ├── select - │ ├── columns: l_suppkey:10(int!null) sum:25(float!null) - │ ├── key: (10) - │ ├── fd: (10)-->(25) - │ ├── group-by - │ │ ├── columns: l_suppkey:10(int!null) sum:25(float) - │ │ ├── grouping columns: l_suppkey:10(int!null) - │ │ ├── key: (10) - │ │ ├── fd: (10)-->(25) - │ │ ├── project - │ │ │ ├── columns: column24:24(float) l_suppkey:10(int!null) - │ │ │ ├── index-join lineitem - │ │ │ │ ├── columns: l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) - │ │ │ │ └── scan lineitem@l_sd - │ │ │ │ ├── columns: l_orderkey:8(int!null) l_linenumber:11(int!null) l_shipdate:18(date!null) - │ │ │ │ ├── constraint: /18/8/11: [/'1996-01-01' - /'1996-03-31'] - │ │ │ │ ├── key: (8,11) - │ │ │ │ └── fd: (8,11)-->(18) - │ │ │ └── projections - │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(13,14)] - │ │ └── aggregations - │ │ └── sum [type=float, outer=(24)] - │ │ └── variable: column24 [type=float] - │ └── filters - │ └── eq [type=bool, outer=(25), constraints=(/25: (/NULL - ])] - │ ├── variable: sum [type=float] - │ └── subquery [type=float] - │ └── scalar-group-by - │ ├── columns: max:44(float) - │ ├── cardinality: [1 - 1] - │ ├── key: () - │ ├── fd: ()-->(44) - │ ├── group-by - │ │ ├── columns: l_suppkey:28(int!null) sum:43(float) - │ │ ├── grouping columns: l_suppkey:28(int!null) - │ │ ├── key: (28) - │ │ ├── fd: (28)-->(43) - │ │ ├── project - │ │ │ ├── columns: column42:42(float) l_suppkey:28(int!null) - │ │ │ ├── index-join lineitem - │ │ │ │ ├── columns: l_suppkey:28(int!null) l_extendedprice:31(float!null) l_discount:32(float!null) l_shipdate:36(date!null) - │ │ │ │ └── scan lineitem@l_sd - │ │ │ │ ├── columns: l_orderkey:26(int!null) l_linenumber:29(int!null) l_shipdate:36(date!null) - │ │ │ │ ├── constraint: /36/26/29: [/'1996-01-01' - /'1996-03-31'] - │ │ │ │ ├── key: (26,29) - │ │ │ │ └── fd: (26,29)-->(36) - │ │ │ └── projections - │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(31,32)] - │ │ └── aggregations - │ │ └── sum [type=float, outer=(42)] - │ │ └── variable: column42 [type=float] - │ └── aggregations - │ └── max [type=float, outer=(43)] - │ └── variable: sum [type=float] - └── filters (true) + └── inner-join (merge) + ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_phone:5(string!null) l_suppkey:10(int!null) sum:25(float!null) + ├── left ordering: +1 + ├── right ordering: +10 + ├── key: (10) + ├── fd: (1)-->(2,3,5), (10)-->(25), (1)==(10), (10)==(1) + ├── ordering: +(1|10) [actual: +1] + ├── scan supplier + │ ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_phone:5(string!null) + │ ├── key: (1) + │ ├── fd: (1)-->(2,3,5) + │ └── ordering: +1 + ├── sort + │ ├── columns: l_suppkey:10(int!null) sum:25(float!null) + │ ├── key: (10) + │ ├── fd: (10)-->(25) + │ ├── ordering: +10 + │ └── select + │ ├── columns: l_suppkey:10(int!null) sum:25(float!null) + │ ├── key: (10) + │ ├── fd: (10)-->(25) + │ ├── group-by + │ │ ├── columns: l_suppkey:10(int!null) sum:25(float) + │ │ ├── grouping columns: l_suppkey:10(int!null) + │ │ ├── key: (10) + │ │ ├── fd: (10)-->(25) + │ │ ├── project + │ │ │ ├── columns: column24:24(float) l_suppkey:10(int!null) + │ │ │ ├── index-join lineitem + │ │ │ │ ├── columns: l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) + │ │ │ │ └── scan lineitem@l_sd + │ │ │ │ ├── columns: l_orderkey:8(int!null) l_linenumber:11(int!null) l_shipdate:18(date!null) + │ │ │ │ ├── constraint: /18/8/11: [/'1996-01-01' - /'1996-03-31'] + │ │ │ │ ├── key: (8,11) + │ │ │ │ └── fd: (8,11)-->(18) + │ │ │ └── projections + │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(13,14)] + │ │ └── aggregations + │ │ └── sum [type=float, outer=(24)] + │ │ └── variable: column24 [type=float] + │ └── filters + │ └── eq [type=bool, outer=(25), constraints=(/25: (/NULL - ])] + │ ├── variable: sum [type=float] + │ └── subquery [type=float] + │ └── scalar-group-by + │ ├── columns: max:44(float) + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(44) + │ ├── group-by + │ │ ├── columns: l_suppkey:28(int!null) sum:43(float) + │ │ ├── grouping columns: l_suppkey:28(int!null) + │ │ ├── key: (28) + │ │ ├── fd: (28)-->(43) + │ │ ├── project + │ │ │ ├── columns: column42:42(float) l_suppkey:28(int!null) + │ │ │ ├── index-join lineitem + │ │ │ │ ├── columns: l_suppkey:28(int!null) l_extendedprice:31(float!null) l_discount:32(float!null) l_shipdate:36(date!null) + │ │ │ │ └── scan lineitem@l_sd + │ │ │ │ ├── columns: l_orderkey:26(int!null) l_linenumber:29(int!null) l_shipdate:36(date!null) + │ │ │ │ ├── constraint: /36/26/29: [/'1996-01-01' - /'1996-03-31'] + │ │ │ │ ├── key: (26,29) + │ │ │ │ └── fd: (26,29)-->(36) + │ │ │ └── projections + │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(31,32)] + │ │ └── aggregations + │ │ └── sum [type=float, outer=(42)] + │ │ └── variable: column42 [type=float] + │ └── aggregations + │ └── max [type=float, outer=(43)] + │ └── variable: sum [type=float] + └── filters (true) # -------------------------------------------------- # Q16 @@ -2204,10 +2369,6 @@ limit │ ├── inner-join │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) o_orderkey:9(int!null) o_custkey:10(int!null) o_totalprice:12(float!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_quantity:22(float!null) │ │ ├── fd: (1)-->(2), (9)-->(10,12,13), (9)==(18), (18)==(9), (1)==(10), (10)==(1) - │ │ ├── scan customer - │ │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) - │ │ │ ├── key: (1) - │ │ │ └── fd: (1)-->(2) │ │ ├── inner-join (merge) │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_totalprice:12(float!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_quantity:22(float!null) │ │ │ ├── left ordering: +9 @@ -2249,6 +2410,10 @@ limit │ │ │ │ ├── columns: l_orderkey:18(int!null) l_quantity:22(float!null) │ │ │ │ └── ordering: +18 │ │ │ └── filters (true) + │ │ ├── scan customer + │ │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) + │ │ │ ├── key: (1) + │ │ │ └── fd: (1)-->(2) │ │ └── filters │ │ └── c_custkey = o_custkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] │ └── aggregations @@ -2546,20 +2711,20 @@ limit │ ├── grouping columns: s_name:2(string!null) │ ├── key: (2) │ ├── fd: (2)-->(69) - │ ├── inner-join + │ ├── inner-join (lookup supplier) │ │ ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_nationkey:4(int!null) l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) o_orderkey:24(int!null) o_orderstatus:26(string!null) n_nationkey:33(int!null) n_name:34(string!null) + │ │ ├── key columns: [10] = [1] │ │ ├── fd: ()-->(26,34), (1)-->(2,4), (8)==(24), (24)==(8), (1)==(10), (10)==(1), (4)==(33), (33)==(4) - │ │ ├── scan supplier - │ │ │ ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_nationkey:4(int!null) - │ │ │ ├── key: (1) - │ │ │ └── fd: (1)-->(2,4) - │ │ ├── inner-join + │ │ ├── inner-join (merge) │ │ │ ├── columns: l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) o_orderkey:24(int!null) o_orderstatus:26(string!null) n_nationkey:33(int!null) n_name:34(string!null) + │ │ │ ├── left ordering: +8 + │ │ │ ├── right ordering: +24 │ │ │ ├── fd: ()-->(26,34), (8)==(24), (24)==(8) │ │ │ ├── semi-join (merge) │ │ │ │ ├── columns: l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) │ │ │ │ ├── left ordering: +8 │ │ │ │ ├── right ordering: +37 + │ │ │ │ ├── ordering: +8 │ │ │ │ ├── anti-join (merge) │ │ │ │ │ ├── columns: l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) │ │ │ │ │ ├── left ordering: +8 @@ -2594,35 +2759,38 @@ limit │ │ │ │ │ └── ordering: +37 │ │ │ │ └── filters │ │ │ │ └── l2.l_suppkey != l1.l_suppkey [type=bool, outer=(10,39), constraints=(/10: (/NULL - ]; /39: (/NULL - ])] - │ │ │ ├── inner-join + │ │ │ ├── sort │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) n_nationkey:33(int!null) n_name:34(string!null) │ │ │ │ ├── key: (24,33) │ │ │ │ ├── fd: ()-->(26,34) - │ │ │ │ ├── select - │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) - │ │ │ │ │ ├── key: (24) - │ │ │ │ │ ├── fd: ()-->(26) - │ │ │ │ │ ├── scan orders - │ │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) - │ │ │ │ │ │ ├── key: (24) - │ │ │ │ │ │ └── fd: (24)-->(26) - │ │ │ │ │ └── filters - │ │ │ │ │ └── o_orderstatus = 'F' [type=bool, outer=(26), constraints=(/26: [/'F' - /'F']; tight), fd=()-->(26)] - │ │ │ │ ├── select - │ │ │ │ │ ├── columns: n_nationkey:33(int!null) n_name:34(string!null) - │ │ │ │ │ ├── key: (33) - │ │ │ │ │ ├── fd: ()-->(34) - │ │ │ │ │ ├── scan nation - │ │ │ │ │ │ ├── columns: n_nationkey:33(int!null) n_name:34(string!null) - │ │ │ │ │ │ ├── key: (33) - │ │ │ │ │ │ └── fd: (33)-->(34) - │ │ │ │ │ └── filters - │ │ │ │ │ └── n_name = 'SAUDI ARABIA' [type=bool, outer=(34), constraints=(/34: [/'SAUDI ARABIA' - /'SAUDI ARABIA']; tight), fd=()-->(34)] - │ │ │ │ └── filters (true) - │ │ │ └── filters - │ │ │ └── o_orderkey = l1.l_orderkey [type=bool, outer=(8,24), constraints=(/8: (/NULL - ]; /24: (/NULL - ]), fd=(8)==(24), (24)==(8)] + │ │ │ │ ├── ordering: +24 opt(26,34) [actual: +24] + │ │ │ │ └── inner-join + │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) n_nationkey:33(int!null) n_name:34(string!null) + │ │ │ │ ├── key: (24,33) + │ │ │ │ ├── fd: ()-->(26,34) + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) + │ │ │ │ │ ├── key: (24) + │ │ │ │ │ ├── fd: ()-->(26) + │ │ │ │ │ ├── scan orders + │ │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) + │ │ │ │ │ │ ├── key: (24) + │ │ │ │ │ │ └── fd: (24)-->(26) + │ │ │ │ │ └── filters + │ │ │ │ │ └── o_orderstatus = 'F' [type=bool, outer=(26), constraints=(/26: [/'F' - /'F']; tight), fd=()-->(26)] + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: n_nationkey:33(int!null) n_name:34(string!null) + │ │ │ │ │ ├── key: (33) + │ │ │ │ │ ├── fd: ()-->(34) + │ │ │ │ │ ├── scan nation + │ │ │ │ │ │ ├── columns: n_nationkey:33(int!null) n_name:34(string!null) + │ │ │ │ │ │ ├── key: (33) + │ │ │ │ │ │ └── fd: (33)-->(34) + │ │ │ │ │ └── filters + │ │ │ │ │ └── n_name = 'SAUDI ARABIA' [type=bool, outer=(34), constraints=(/34: [/'SAUDI ARABIA' - /'SAUDI ARABIA']; tight), fd=()-->(34)] + │ │ │ │ └── filters (true) + │ │ │ └── filters (true) │ │ └── filters - │ │ ├── s_suppkey = l1.l_suppkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] │ │ └── s_nationkey = n_nationkey [type=bool, outer=(4,33), constraints=(/4: (/NULL - ]; /33: (/NULL - ]), fd=(4)==(33), (33)==(4)] │ └── aggregations │ └── count-rows [type=int] diff --git a/pkg/sql/opt/xform/testdata/external/tpch-no-stats b/pkg/sql/opt/xform/testdata/external/tpch-no-stats new file mode 100644 index 000000000000..ce74518b703e --- /dev/null +++ b/pkg/sql/opt/xform/testdata/external/tpch-no-stats @@ -0,0 +1,2737 @@ +exec-ddl +CREATE TABLE public.region +( + r_regionkey int PRIMARY KEY, + r_name char(25) NOT NULL, + r_comment varchar(152) +); +---- +TABLE region + ├── r_regionkey int not null + ├── r_name string not null + ├── r_comment string + └── INDEX primary + └── r_regionkey int not null + +exec-ddl +CREATE TABLE public.nation +( + n_nationkey int PRIMARY KEY, + n_name char(25) NOT NULL, + n_regionkey int NOT NULL, + n_comment varchar(152), + INDEX n_rk (n_regionkey ASC), + CONSTRAINT nation_fkey_region FOREIGN KEY (n_regionkey) references public.region (r_regionkey) +); +---- +TABLE nation + ├── n_nationkey int not null + ├── n_name string not null + ├── n_regionkey int not null + ├── n_comment string + ├── INDEX primary + │ └── n_nationkey int not null + ├── INDEX n_rk + │ ├── n_regionkey int not null + │ └── n_nationkey int not null + └── FOREIGN KEY (n_regionkey) REFERENCES t.public.region (r_regionkey) + +exec-ddl +CREATE TABLE public.supplier +( + s_suppkey int PRIMARY KEY, + s_name char(25) NOT NULL, + s_address varchar(40) NOT NULL, + s_nationkey int NOT NULL, + s_phone char(15) NOT NULL, + s_acctbal float NOT NULL, + s_comment varchar(101) NOT NULL, + INDEX s_nk (s_nationkey ASC), + CONSTRAINT supplier_fkey_nation FOREIGN KEY (s_nationkey) references public.nation (n_nationkey) +); +---- +TABLE supplier + ├── s_suppkey int not null + ├── s_name string not null + ├── s_address string not null + ├── s_nationkey int not null + ├── s_phone string not null + ├── s_acctbal float not null + ├── s_comment string not null + ├── INDEX primary + │ └── s_suppkey int not null + ├── INDEX s_nk + │ ├── s_nationkey int not null + │ └── s_suppkey int not null + └── FOREIGN KEY (s_nationkey) REFERENCES t.public.nation (n_nationkey) + +exec-ddl +CREATE TABLE public.part +( + p_partkey int PRIMARY KEY, + p_name varchar(55) NOT NULL, + p_mfgr char(25) NOT NULL, + p_brand char(10) NOT NULL, + p_type varchar(25) NOT NULL, + p_size int NOT NULL, + p_container char(10) NOT NULL, + p_retailprice float NOT NULL, + p_comment varchar(23) NOT NULL +); +---- +TABLE part + ├── p_partkey int not null + ├── p_name string not null + ├── p_mfgr string not null + ├── p_brand string not null + ├── p_type string not null + ├── p_size int not null + ├── p_container string not null + ├── p_retailprice float not null + ├── p_comment string not null + └── INDEX primary + └── p_partkey int not null + +exec-ddl +CREATE TABLE public.partsupp +( + ps_partkey int NOT NULL, + ps_suppkey int NOT NULL, + ps_availqty int NOT NULL, + ps_supplycost float NOT NULL, + ps_comment varchar(199) NOT NULL, + PRIMARY KEY (ps_partkey, ps_suppkey), + INDEX ps_sk (ps_suppkey ASC), + CONSTRAINT partsupp_fkey_part FOREIGN KEY (ps_partkey) references public.part (p_partkey), + CONSTRAINT partsupp_fkey_supplier FOREIGN KEY (ps_suppkey) references public.supplier (s_suppkey) +); +---- +TABLE partsupp + ├── ps_partkey int not null + ├── ps_suppkey int not null + ├── ps_availqty int not null + ├── ps_supplycost float not null + ├── ps_comment string not null + ├── INDEX primary + │ ├── ps_partkey int not null + │ └── ps_suppkey int not null + ├── INDEX ps_sk + │ ├── ps_suppkey int not null + │ └── ps_partkey int not null + ├── FOREIGN KEY (ps_partkey) REFERENCES t.public.part (p_partkey) + └── FOREIGN KEY (ps_suppkey) REFERENCES t.public.supplier (s_suppkey) + +exec-ddl +CREATE TABLE public.customer +( + c_custkey int PRIMARY KEY, + c_name varchar(25) NOT NULL, + c_address varchar(40) NOT NULL, + c_nationkey int NOT NULL NOT NULL, + c_phone char(15) NOT NULL, + c_acctbal float NOT NULL, + c_mktsegment char(10) NOT NULL, + c_comment varchar(117) NOT NULL, + INDEX c_nk (c_nationkey ASC), + CONSTRAINT customer_fkey_nation FOREIGN KEY (c_nationkey) references public.nation (n_nationkey) +); +---- +TABLE customer + ├── c_custkey int not null + ├── c_name string not null + ├── c_address string not null + ├── c_nationkey int not null + ├── c_phone string not null + ├── c_acctbal float not null + ├── c_mktsegment string not null + ├── c_comment string not null + ├── INDEX primary + │ └── c_custkey int not null + ├── INDEX c_nk + │ ├── c_nationkey int not null + │ └── c_custkey int not null + └── FOREIGN KEY (c_nationkey) REFERENCES t.public.nation (n_nationkey) + +exec-ddl +CREATE TABLE public.orders +( + o_orderkey int PRIMARY KEY, + o_custkey int NOT NULL, + o_orderstatus char(1) NOT NULL, + o_totalprice float NOT NULL, + o_orderdate date NOT NULL, + o_orderpriority char(15) NOT NULL, + o_clerk char(15) NOT NULL, + o_shippriority int NOT NULL, + o_comment varchar(79) NOT NULL, + INDEX o_ck (o_custkey ASC), + INDEX o_od (o_orderdate ASC), + CONSTRAINT orders_fkey_customer FOREIGN KEY (o_custkey) references public.customer (c_custkey) +); +---- +TABLE orders + ├── o_orderkey int not null + ├── o_custkey int not null + ├── o_orderstatus string not null + ├── o_totalprice float not null + ├── o_orderdate date not null + ├── o_orderpriority string not null + ├── o_clerk string not null + ├── o_shippriority int not null + ├── o_comment string not null + ├── INDEX primary + │ └── o_orderkey int not null + ├── INDEX o_ck + │ ├── o_custkey int not null + │ └── o_orderkey int not null + ├── INDEX o_od + │ ├── o_orderdate date not null + │ └── o_orderkey int not null + └── FOREIGN KEY (o_custkey) REFERENCES t.public.customer (c_custkey) + +exec-ddl +CREATE TABLE public.lineitem +( + l_orderkey int NOT NULL, + l_partkey int NOT NULL, + l_suppkey int NOT NULL, + l_linenumber int NOT NULL, + l_quantity float NOT NULL, + l_extendedprice float NOT NULL, + l_discount float NOT NULL, + l_tax float NOT NULL, + l_returnflag char(1) NOT NULL, + l_linestatus char(1) NOT NULL, + l_shipdate date NOT NULL, + l_commitdate date NOT NULL, + l_receiptdate date NOT NULL, + l_shipinstruct char(25) NOT NULL, + l_shipmode char(10) NOT NULL, + l_comment varchar(44) NOT NULL, + PRIMARY KEY (l_orderkey, l_linenumber), + INDEX l_ok (l_orderkey ASC), + INDEX l_pk (l_partkey ASC), + INDEX l_sk (l_suppkey ASC), + INDEX l_sd (l_shipdate ASC), + INDEX l_cd (l_commitdate ASC), + INDEX l_rd (l_receiptdate ASC), + INDEX l_pk_sk (l_partkey ASC, l_suppkey ASC), + INDEX l_sk_pk (l_suppkey ASC, l_partkey ASC), + CONSTRAINT lineitem_fkey_orders FOREIGN KEY (l_orderkey) references public.orders (o_orderkey), + CONSTRAINT lineitem_fkey_part FOREIGN KEY (l_partkey) references public.part (p_partkey), + CONSTRAINT lineitem_fkey_supplier FOREIGN KEY (l_suppkey) references public.supplier (s_suppkey), + CONSTRAINT lineitem_fkey_partsupp FOREIGN KEY (l_partkey, l_suppkey) references public.partsupp (ps_partkey, ps_suppkey) +); +---- +TABLE lineitem + ├── l_orderkey int not null + ├── l_partkey int not null + ├── l_suppkey int not null + ├── l_linenumber int not null + ├── l_quantity float not null + ├── l_extendedprice float not null + ├── l_discount float not null + ├── l_tax float not null + ├── l_returnflag string not null + ├── l_linestatus string not null + ├── l_shipdate date not null + ├── l_commitdate date not null + ├── l_receiptdate date not null + ├── l_shipinstruct string not null + ├── l_shipmode string not null + ├── l_comment string not null + ├── INDEX primary + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── INDEX l_ok + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── INDEX l_pk + │ ├── l_partkey int not null + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── INDEX l_sk + │ ├── l_suppkey int not null + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── INDEX l_sd + │ ├── l_shipdate date not null + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── INDEX l_cd + │ ├── l_commitdate date not null + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── INDEX l_rd + │ ├── l_receiptdate date not null + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── INDEX l_pk_sk + │ ├── l_partkey int not null + │ ├── l_suppkey int not null + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── INDEX l_sk_pk + │ ├── l_suppkey int not null + │ ├── l_partkey int not null + │ ├── l_orderkey int not null + │ └── l_linenumber int not null + ├── FOREIGN KEY (l_orderkey) REFERENCES t.public.orders (o_orderkey) + ├── FOREIGN KEY (l_partkey) REFERENCES t.public.part (p_partkey) + ├── FOREIGN KEY (l_suppkey) REFERENCES t.public.supplier (s_suppkey) + └── FOREIGN KEY (l_partkey, l_suppkey) REFERENCES t.public.partsupp (ps_partkey, ps_suppkey) + +# -------------------------------------------------- +# Q1 +# Pricing Summary Report +# Reports the amount of business that was billed, shipped, and returned. +# +# Provides a summary pricing report for all lineitems shipped as of a given +# date. The date is within 60 - 120 days of the greatest ship date contained in +# the database. The query lists totals for extended price, discounted extended +# price, discounted extended price plus tax, average quantity, average extended +# price, and average discount. These aggregates are grouped by RETURNFLAG and +# LINESTATUS, and listed in ascending order of RETURNFLAG and LINESTATUS. A +# count of the number of lineitems in each group is included. +# -------------------------------------------------- +opt +SELECT + l_returnflag, + l_linestatus, + sum(l_quantity) AS sum_qty, + sum(l_extendedprice) AS sum_base_price, + sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + avg(l_quantity) AS avg_qty, + avg(l_extendedprice) AS avg_price, + avg(l_discount) AS avg_disc, + count(*) AS count_order +FROM + lineitem +WHERE + l_shipdate <= DATE '1998-12-01' - INTERVAL '90' DAY +GROUP BY + l_returnflag, + l_linestatus +ORDER BY + l_returnflag, + l_linestatus; +---- +group-by + ├── columns: l_returnflag:9(string!null) l_linestatus:10(string!null) sum_qty:17(float) sum_base_price:18(float) sum_disc_price:20(float) sum_charge:22(float) avg_qty:23(float) avg_price:24(float) avg_disc:25(float) count_order:26(int) + ├── grouping columns: l_returnflag:9(string!null) l_linestatus:10(string!null) + ├── key: (9,10) + ├── fd: (9,10)-->(17,18,20,22-26) + ├── ordering: +9,+10 + ├── sort + │ ├── columns: l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_returnflag:9(string!null) l_linestatus:10(string!null) column19:19(float) column21:21(float) + │ ├── ordering: +9,+10 + │ └── project + │ ├── columns: column19:19(float) column21:21(float) l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_returnflag:9(string!null) l_linestatus:10(string!null) + │ ├── select + │ │ ├── columns: l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_tax:8(float!null) l_returnflag:9(string!null) l_linestatus:10(string!null) l_shipdate:11(date!null) + │ │ ├── scan lineitem + │ │ │ └── columns: l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_tax:8(float!null) l_returnflag:9(string!null) l_linestatus:10(string!null) l_shipdate:11(date!null) + │ │ └── filters + │ │ └── l_shipdate <= '1998-09-02' [type=bool, outer=(11), constraints=(/11: (/NULL - /'1998-09-02']; tight)] + │ └── projections + │ ├── l_extendedprice * (1.0 - l_discount) [type=float, outer=(6,7)] + │ └── (l_extendedprice * (1.0 - l_discount)) * (l_tax + 1.0) [type=float, outer=(6-8)] + └── aggregations + ├── sum [type=float, outer=(5)] + │ └── variable: l_quantity [type=float] + ├── sum [type=float, outer=(6)] + │ └── variable: l_extendedprice [type=float] + ├── sum [type=float, outer=(19)] + │ └── variable: column19 [type=float] + ├── sum [type=float, outer=(21)] + │ └── variable: column21 [type=float] + ├── avg [type=float, outer=(5)] + │ └── variable: l_quantity [type=float] + ├── avg [type=float, outer=(6)] + │ └── variable: l_extendedprice [type=float] + ├── avg [type=float, outer=(7)] + │ └── variable: l_discount [type=float] + └── count-rows [type=int] + +# -------------------------------------------------- +# Q2 +# Minimum Cost Supplier +# Finds which supplier should be selected to place an order for a given part in +# a given region. +# +# Finds, in a given region, for each part of a certain type and size, the +# supplier who can supply it at minimum cost. If several suppliers in that +# region offer the desired part type and size at the same (minimum) cost, the +# query lists the parts from suppliers with the 100 highest account balances. +# For each supplier, the query lists the supplier's account balance, name and +# nation; the part's number and manufacturer; the supplier's address, phone +# number and comment information. +# +# TODO: +# 1. Join ordering +# 2. Push down equivalent column comparisons +# 3. Allow Select to be pushed below RowNumber used to add key column +# 4. Add decorrelation rule for RowNumber/RowKey +# -------------------------------------------------- +opt +SELECT + s_acctbal, + s_name, + n_name, + p_partkey, + p_mfgr, + s_address, + s_phone, + s_comment +FROM + part, + supplier, + partsupp, + nation, + region +WHERE + p_partkey = ps_partkey + AND s_suppkey = ps_suppkey + AND p_size = 15 + AND p_type LIKE '%BRASS' + AND s_nationkey = n_nationkey + AND n_regionkey = r_regionkey + AND r_name = 'EUROPE' + AND ps_supplycost = ( + SELECT + min(ps_supplycost) + FROM + partsupp, + supplier, + nation, + region + WHERE + p_partkey = ps_partkey + AND s_suppkey = ps_suppkey + AND s_nationkey = n_nationkey + AND n_regionkey = r_regionkey + AND r_name = 'EUROPE' + ) +ORDER BY + s_acctbal DESC, + n_name, + s_name, + p_partkey +LIMIT 100; +---- +project + ├── columns: s_acctbal:15(float) s_name:11(string) n_name:23(string) p_partkey:1(int) p_mfgr:3(string) s_address:12(string) s_phone:14(string) s_comment:16(string) + ├── cardinality: [0 - 100] + ├── fd: (1)-->(3) + ├── ordering: -15,+23,+11,+1 + └── limit + ├── columns: p_partkey:1(int) p_mfgr:3(string) s_name:11(string) s_address:12(string) s_phone:14(string) s_acctbal:15(float) s_comment:16(string) ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) n_name:23(string) min:48(float!null) + ├── internal-ordering: -15,+23,+11,+(1|17) + ├── cardinality: [0 - 100] + ├── key: (17,18) + ├── fd: (1)-->(3), (17,18)-->(1,3,11,12,14-16,20,23,48), (1)==(17), (17)==(1), (18)-->(11,12,14-16,23), (20)==(48), (48)==(20) + ├── ordering: -15,+23,+11,+(1|17) [actual: -15,+23,+11,+1] + ├── sort + │ ├── columns: p_partkey:1(int) p_mfgr:3(string) s_name:11(string) s_address:12(string) s_phone:14(string) s_acctbal:15(float) s_comment:16(string) ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) n_name:23(string) min:48(float!null) + │ ├── key: (17,18) + │ ├── fd: (1)-->(3), (17,18)-->(1,3,11,12,14-16,20,23,48), (1)==(17), (17)==(1), (18)-->(11,12,14-16,23), (20)==(48), (48)==(20) + │ ├── ordering: -15,+23,+11,+(1|17) [actual: -15,+23,+11,+1] + │ └── select + │ ├── columns: p_partkey:1(int) p_mfgr:3(string) s_name:11(string) s_address:12(string) s_phone:14(string) s_acctbal:15(float) s_comment:16(string) ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) n_name:23(string) min:48(float!null) + │ ├── key: (17,18) + │ ├── fd: (1)-->(3), (17,18)-->(1,3,11,12,14-16,20,23,48), (1)==(17), (17)==(1), (18)-->(11,12,14-16,23), (20)==(48), (48)==(20) + │ ├── group-by + │ │ ├── columns: p_partkey:1(int) p_mfgr:3(string) s_name:11(string) s_address:12(string) s_phone:14(string) s_acctbal:15(float) s_comment:16(string) ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float) n_name:23(string) min:48(float) + │ │ ├── grouping columns: ps_partkey:17(int!null) ps_suppkey:18(int!null) + │ │ ├── key: (17,18) + │ │ ├── fd: (1)-->(3), (17,18)-->(1,3,11,12,14-16,20,23,48), (1)==(17), (17)==(1), (18)-->(11,12,14-16,23) + │ │ ├── inner-join + │ │ │ ├── columns: p_partkey:1(int!null) p_mfgr:3(string!null) p_type:5(string!null) p_size:6(int!null) s_suppkey:10(int!null) s_name:11(string!null) s_address:12(string!null) s_nationkey:13(int!null) s_phone:14(string!null) s_acctbal:15(float!null) s_comment:16(string!null) ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) n_nationkey:22(int!null) n_name:23(string!null) n_regionkey:24(int!null) r_regionkey:26(int!null) r_name:27(string!null) ps_partkey:29(int!null) ps_suppkey:30(int!null) ps_supplycost:32(float!null) s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ ├── key: (18,29,34) + │ │ │ ├── fd: ()-->(6,27,46), (1)-->(3,5), (10)-->(11-16), (17,18)-->(20), (22)-->(23,24), (24)==(26), (26)==(24), (10)==(18), (18)==(10), (13)==(22), (22)==(13), (1)==(17,29), (17)==(1,29), (29,30)-->(32), (34)-->(37), (41)-->(43), (43)==(45), (45)==(43), (37)==(41), (41)==(37), (30)==(34), (34)==(30), (29)==(1,17) + │ │ │ ├── inner-join + │ │ │ │ ├── columns: ps_partkey:29(int!null) ps_suppkey:30(int!null) ps_supplycost:32(float!null) s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ ├── key: (29,34) + │ │ │ │ ├── fd: ()-->(46), (29,30)-->(32), (34)-->(37), (41)-->(43), (43)==(45), (45)==(43), (37)==(41), (41)==(37), (30)==(34), (34)==(30) + │ │ │ │ ├── scan partsupp + │ │ │ │ │ ├── columns: ps_partkey:29(int!null) ps_suppkey:30(int!null) ps_supplycost:32(float!null) + │ │ │ │ │ ├── key: (29,30) + │ │ │ │ │ └── fd: (29,30)-->(32) + │ │ │ │ ├── inner-join + │ │ │ │ │ ├── columns: s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ ├── key: (34) + │ │ │ │ │ ├── fd: ()-->(46), (34)-->(37), (41)-->(43), (43)==(45), (45)==(43), (37)==(41), (41)==(37) + │ │ │ │ │ ├── scan supplier@s_nk + │ │ │ │ │ │ ├── columns: s_suppkey:34(int!null) s_nationkey:37(int!null) + │ │ │ │ │ │ ├── key: (34) + │ │ │ │ │ │ └── fd: (34)-->(37) + │ │ │ │ │ ├── inner-join (lookup nation@n_rk) + │ │ │ │ │ │ ├── columns: n_nationkey:41(int!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ │ ├── key columns: [45] = [43] + │ │ │ │ │ │ ├── key: (41) + │ │ │ │ │ │ ├── fd: ()-->(46), (41)-->(43), (43)==(45), (45)==(43) + │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ │ │ ├── key: (45) + │ │ │ │ │ │ │ ├── fd: ()-->(46) + │ │ │ │ │ │ │ ├── scan region + │ │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ │ │ │ ├── key: (45) + │ │ │ │ │ │ │ │ └── fd: (45)-->(46) + │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ └── r_name = 'EUROPE' [type=bool, outer=(46), constraints=(/46: [/'EUROPE' - /'EUROPE']; tight), fd=()-->(46)] + │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ └── filters + │ │ │ │ │ └── s_nationkey = n_nationkey [type=bool, outer=(37,41), constraints=(/37: (/NULL - ]; /41: (/NULL - ]), fd=(37)==(41), (41)==(37)] + │ │ │ │ └── filters + │ │ │ │ └── s_suppkey = ps_suppkey [type=bool, outer=(30,34), constraints=(/30: (/NULL - ]; /34: (/NULL - ]), fd=(30)==(34), (34)==(30)] + │ │ │ ├── inner-join + │ │ │ │ ├── columns: p_partkey:1(int!null) p_mfgr:3(string!null) p_type:5(string!null) p_size:6(int!null) s_suppkey:10(int!null) s_name:11(string!null) s_address:12(string!null) s_nationkey:13(int!null) s_phone:14(string!null) s_acctbal:15(float!null) s_comment:16(string!null) ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) n_nationkey:22(int!null) n_name:23(string!null) n_regionkey:24(int!null) r_regionkey:26(int!null) r_name:27(string!null) + │ │ │ │ ├── key: (17,18) + │ │ │ │ ├── fd: ()-->(6,27), (1)-->(3,5), (10)-->(11-16), (17,18)-->(20), (22)-->(23,24), (24)==(26), (26)==(24), (10)==(18), (18)==(10), (13)==(22), (22)==(13), (1)==(17), (17)==(1) + │ │ │ │ ├── inner-join + │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_name:11(string!null) s_address:12(string!null) s_nationkey:13(int!null) s_phone:14(string!null) s_acctbal:15(float!null) s_comment:16(string!null) ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) n_nationkey:22(int!null) n_name:23(string!null) n_regionkey:24(int!null) r_regionkey:26(int!null) r_name:27(string!null) + │ │ │ │ │ ├── key: (17,18) + │ │ │ │ │ ├── fd: ()-->(27), (10)-->(11-16), (17,18)-->(20), (22)-->(23,24), (24)==(26), (26)==(24), (10)==(18), (18)==(10), (13)==(22), (22)==(13) + │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ ├── columns: ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) n_nationkey:22(int!null) n_name:23(string!null) n_regionkey:24(int!null) r_regionkey:26(int!null) r_name:27(string!null) + │ │ │ │ │ │ ├── key: (17,18,22) + │ │ │ │ │ │ ├── fd: ()-->(27), (17,18)-->(20), (22)-->(23,24), (24)==(26), (26)==(24) + │ │ │ │ │ │ ├── scan partsupp + │ │ │ │ │ │ │ ├── columns: ps_partkey:17(int!null) ps_suppkey:18(int!null) ps_supplycost:20(float!null) + │ │ │ │ │ │ │ ├── key: (17,18) + │ │ │ │ │ │ │ └── fd: (17,18)-->(20) + │ │ │ │ │ │ ├── inner-join (lookup nation) + │ │ │ │ │ │ │ ├── columns: n_nationkey:22(int!null) n_name:23(string!null) n_regionkey:24(int!null) r_regionkey:26(int!null) r_name:27(string!null) + │ │ │ │ │ │ │ ├── key columns: [22] = [22] + │ │ │ │ │ │ │ ├── key: (22) + │ │ │ │ │ │ │ ├── fd: ()-->(27), (22)-->(23,24), (24)==(26), (26)==(24) + │ │ │ │ │ │ │ ├── inner-join (lookup nation@n_rk) + │ │ │ │ │ │ │ │ ├── columns: n_nationkey:22(int!null) n_regionkey:24(int!null) r_regionkey:26(int!null) r_name:27(string!null) + │ │ │ │ │ │ │ │ ├── key columns: [26] = [24] + │ │ │ │ │ │ │ │ ├── key: (22) + │ │ │ │ │ │ │ │ ├── fd: ()-->(27), (22)-->(24), (24)==(26), (26)==(24) + │ │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:26(int!null) r_name:27(string!null) + │ │ │ │ │ │ │ │ │ ├── key: (26) + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(27) + │ │ │ │ │ │ │ │ │ ├── scan region + │ │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:26(int!null) r_name:27(string!null) + │ │ │ │ │ │ │ │ │ │ ├── key: (26) + │ │ │ │ │ │ │ │ │ │ └── fd: (26)-->(27) + │ │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ │ └── r_name = 'EUROPE' [type=bool, outer=(27), constraints=(/27: [/'EUROPE' - /'EUROPE']; tight), fd=()-->(27)] + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ ├── scan supplier + │ │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_name:11(string!null) s_address:12(string!null) s_nationkey:13(int!null) s_phone:14(string!null) s_acctbal:15(float!null) s_comment:16(string!null) + │ │ │ │ │ │ ├── key: (10) + │ │ │ │ │ │ └── fd: (10)-->(11-16) + │ │ │ │ │ └── filters + │ │ │ │ │ ├── s_suppkey = ps_suppkey [type=bool, outer=(10,18), constraints=(/10: (/NULL - ]; /18: (/NULL - ]), fd=(10)==(18), (18)==(10)] + │ │ │ │ │ └── s_nationkey = n_nationkey [type=bool, outer=(13,22), constraints=(/13: (/NULL - ]; /22: (/NULL - ]), fd=(13)==(22), (22)==(13)] + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: p_partkey:1(int!null) p_mfgr:3(string!null) p_type:5(string!null) p_size:6(int!null) + │ │ │ │ │ ├── key: (1) + │ │ │ │ │ ├── fd: ()-->(6), (1)-->(3,5) + │ │ │ │ │ ├── scan part + │ │ │ │ │ │ ├── columns: p_partkey:1(int!null) p_mfgr:3(string!null) p_type:5(string!null) p_size:6(int!null) + │ │ │ │ │ │ ├── key: (1) + │ │ │ │ │ │ └── fd: (1)-->(3,5,6) + │ │ │ │ │ └── filters + │ │ │ │ │ ├── p_size = 15 [type=bool, outer=(6), constraints=(/6: [/15 - /15]; tight), fd=()-->(6)] + │ │ │ │ │ └── p_type LIKE '%BRASS' [type=bool, outer=(5), constraints=(/5: (/NULL - ])] + │ │ │ │ └── filters + │ │ │ │ └── p_partkey = ps_partkey [type=bool, outer=(1,17), constraints=(/1: (/NULL - ]; /17: (/NULL - ]), fd=(1)==(17), (17)==(1)] + │ │ │ └── filters + │ │ │ └── p_partkey = ps_partkey [type=bool, outer=(1,29), constraints=(/1: (/NULL - ]; /29: (/NULL - ]), fd=(1)==(29), (29)==(1)] + │ │ └── aggregations + │ │ ├── min [type=float, outer=(32)] + │ │ │ └── variable: ps_supplycost [type=float] + │ │ ├── const-agg [type=string, outer=(11)] + │ │ │ └── variable: s_name [type=string] + │ │ ├── const-agg [type=string, outer=(12)] + │ │ │ └── variable: s_address [type=string] + │ │ ├── const-agg [type=string, outer=(14)] + │ │ │ └── variable: s_phone [type=string] + │ │ ├── const-agg [type=float, outer=(15)] + │ │ │ └── variable: s_acctbal [type=float] + │ │ ├── const-agg [type=string, outer=(16)] + │ │ │ └── variable: s_comment [type=string] + │ │ ├── const-agg [type=float, outer=(20)] + │ │ │ └── variable: ps_supplycost [type=float] + │ │ ├── const-agg [type=string, outer=(23)] + │ │ │ └── variable: n_name [type=string] + │ │ ├── const-agg [type=string, outer=(3)] + │ │ │ └── variable: p_mfgr [type=string] + │ │ └── const-agg [type=int, outer=(1)] + │ │ └── variable: p_partkey [type=int] + │ └── filters + │ └── ps_supplycost = min [type=bool, outer=(20,48), constraints=(/20: (/NULL - ]; /48: (/NULL - ]), fd=(20)==(48), (48)==(20)] + └── const: 100 [type=int] + +# -------------------------------------------------- +# Q3 +# Shipping Priority +# Retrieves the 10 unshipped orders with the highest value. +# +# Retrieves the shipping priority and potential revenue, defined as the sum of +# l_extendedprice * (1-l_discount), of the orders having the largest revenue +# among those that had not been shipped as of a given date. Orders are listed in +# decreasing order of revenue. If more than 10 unshipped orders exist, only the +# 10 orders with the largest revenue are listed. +# -------------------------------------------------- +opt +SELECT + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) AS revenue, + o_orderdate, + o_shippriority +FROM + customer, + orders, + lineitem +WHERE + c_mktsegment = 'BUILDING' + AND c_custkey = o_custkey + AND l_orderkey = o_orderkey + AND o_orderDATE < DATE '1995-03-15' + AND l_shipdate > DATE '1995-03-15' +GROUP BY + l_orderkey, + o_orderdate, + o_shippriority +ORDER BY + revenue DESC, + o_orderdate +LIMIT 10; +---- +limit + ├── columns: l_orderkey:18(int!null) revenue:35(float) o_orderdate:13(date) o_shippriority:16(int) + ├── internal-ordering: -35,+13 + ├── cardinality: [0 - 10] + ├── key: (18) + ├── fd: (18)-->(13,16,35) + ├── ordering: -35,+13 + ├── sort + │ ├── columns: o_orderdate:13(date) o_shippriority:16(int) l_orderkey:18(int!null) sum:35(float) + │ ├── key: (18) + │ ├── fd: (18)-->(13,16,35) + │ ├── ordering: -35,+13 + │ └── group-by + │ ├── columns: o_orderdate:13(date) o_shippriority:16(int) l_orderkey:18(int!null) sum:35(float) + │ ├── grouping columns: l_orderkey:18(int!null) + │ ├── key: (18) + │ ├── fd: (18)-->(13,16,35) + │ ├── project + │ │ ├── columns: column34:34(float) o_orderdate:13(date!null) o_shippriority:16(int!null) l_orderkey:18(int!null) + │ │ ├── fd: (18)-->(13,16) + │ │ ├── inner-join + │ │ │ ├── columns: c_custkey:1(int!null) c_mktsegment:7(string!null) o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) o_shippriority:16(int!null) l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_shipdate:28(date!null) + │ │ │ ├── fd: ()-->(7), (9)-->(10,13,16), (9)==(18), (18)==(9), (1)==(10), (10)==(1) + │ │ │ ├── inner-join (merge) + │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) o_shippriority:16(int!null) l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_shipdate:28(date!null) + │ │ │ │ ├── left ordering: +9 + │ │ │ │ ├── right ordering: +18 + │ │ │ │ ├── fd: (9)-->(10,13,16), (9)==(18), (18)==(9) + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) o_shippriority:16(int!null) + │ │ │ │ │ ├── key: (9) + │ │ │ │ │ ├── fd: (9)-->(10,13,16) + │ │ │ │ │ ├── ordering: +9 + │ │ │ │ │ ├── scan orders + │ │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) o_shippriority:16(int!null) + │ │ │ │ │ │ ├── key: (9) + │ │ │ │ │ │ ├── fd: (9)-->(10,13,16) + │ │ │ │ │ │ └── ordering: +9 + │ │ │ │ │ └── filters + │ │ │ │ │ └── o_orderdate < '1995-03-15' [type=bool, outer=(13), constraints=(/13: (/NULL - /'1995-03-14']; tight)] + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_shipdate:28(date!null) + │ │ │ │ │ ├── ordering: +18 + │ │ │ │ │ ├── scan lineitem + │ │ │ │ │ │ ├── columns: l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_shipdate:28(date!null) + │ │ │ │ │ │ └── ordering: +18 + │ │ │ │ │ └── filters + │ │ │ │ │ └── l_shipdate > '1995-03-15' [type=bool, outer=(28), constraints=(/28: [/'1995-03-16' - ]; tight)] + │ │ │ │ └── filters (true) + │ │ │ ├── select + │ │ │ │ ├── columns: c_custkey:1(int!null) c_mktsegment:7(string!null) + │ │ │ │ ├── key: (1) + │ │ │ │ ├── fd: ()-->(7) + │ │ │ │ ├── scan customer + │ │ │ │ │ ├── columns: c_custkey:1(int!null) c_mktsegment:7(string!null) + │ │ │ │ │ ├── key: (1) + │ │ │ │ │ └── fd: (1)-->(7) + │ │ │ │ └── filters + │ │ │ │ └── c_mktsegment = 'BUILDING' [type=bool, outer=(7), constraints=(/7: [/'BUILDING' - /'BUILDING']; tight), fd=()-->(7)] + │ │ │ └── filters + │ │ │ └── c_custkey = o_custkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ └── projections + │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(23,24)] + │ └── aggregations + │ ├── sum [type=float, outer=(34)] + │ │ └── variable: column34 [type=float] + │ ├── const-agg [type=date, outer=(13)] + │ │ └── variable: o_orderdate [type=date] + │ └── const-agg [type=int, outer=(16)] + │ └── variable: o_shippriority [type=int] + └── const: 10 [type=int] + +# -------------------------------------------------- +# Q4 +# Order Priority Checking +# Determines how well the order priority system is working and gives an +# assessment of customer satisfaction. +# +# Counts the number of orders ordered in a given quarter of a given year in +# which at least one lineitem was received by the customer later than its +# committed date. The query lists the count of such orders for each order +# priority sorted in ascending priority order. +# -------------------------------------------------- +opt +SELECT + o_orderpriority, + count(*) AS order_count +FROM + orders +WHERE + o_orderdate >= DATE '1993-07-01' + AND o_orderdate < DATE '1993-07-01' + INTERVAL '3' MONTH + AND EXISTS ( + SELECT + * + FROM + lineitem + WHERE + l_orderkey = o_orderkey + AND l_commitDATE < l_receiptdate + ) +GROUP BY + o_orderpriority +ORDER BY + o_orderpriority; +---- +sort + ├── columns: o_orderpriority:6(string!null) order_count:26(int) + ├── key: (6) + ├── fd: (6)-->(26) + ├── ordering: +6 + └── group-by + ├── columns: o_orderpriority:6(string!null) count_rows:26(int) + ├── grouping columns: o_orderpriority:6(string!null) + ├── key: (6) + ├── fd: (6)-->(26) + ├── semi-join + │ ├── columns: o_orderkey:1(int!null) o_orderdate:5(date!null) o_orderpriority:6(string!null) + │ ├── key: (1) + │ ├── fd: (1)-->(5,6) + │ ├── index-join orders + │ │ ├── columns: o_orderkey:1(int!null) o_orderdate:5(date!null) o_orderpriority:6(string!null) + │ │ ├── key: (1) + │ │ ├── fd: (1)-->(5,6) + │ │ └── scan orders@o_od + │ │ ├── columns: o_orderkey:1(int!null) o_orderdate:5(date!null) + │ │ ├── constraint: /5/1: [/'1993-07-01' - /'1993-09-30'] + │ │ ├── key: (1) + │ │ └── fd: (1)-->(5) + │ ├── select + │ │ ├── columns: l_orderkey:10(int!null) l_commitdate:21(date!null) l_receiptdate:22(date!null) + │ │ ├── scan lineitem + │ │ │ └── columns: l_orderkey:10(int!null) l_commitdate:21(date!null) l_receiptdate:22(date!null) + │ │ └── filters + │ │ └── l_commitdate < l_receiptdate [type=bool, outer=(21,22), constraints=(/21: (/NULL - ]; /22: (/NULL - ])] + │ └── filters + │ └── l_orderkey = o_orderkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + └── aggregations + └── count-rows [type=int] + +# -------------------------------------------------- +# Q5 +# Local Supplier Volume +# Lists the revenue volume done through local suppliers. +# +# Lists for each nation in a region the revenue volume that resulted from +# lineitem transactions in which the customer ordering parts and the supplier +# filling them were both within that nation. The query is run in order to +# determine whether to institute local distribution centers in a given region. +# The query considers only parts ordered in a given year. The query displays the +# nations and revenue volume in descending order by revenue. Revenue volume for +# all qualifying lineitems in a particular nation is defined as +# sum(l_extendedprice * (1 - l_discount)). +# +# TODO: +# 1. Join ordering +# -------------------------------------------------- +opt +SELECT + n_name, + sum(l_extendedprice * (1 - l_discount)) AS revenue +FROM + customer, + orders, + lineitem, + supplier, + nation, + region +WHERE + c_custkey = o_custkey + AND l_orderkey = o_orderkey + AND l_suppkey = s_suppkey + AND c_nationkey = s_nationkey + AND s_nationkey = n_nationkey + AND n_regionkey = r_regionkey + AND r_name = 'ASIA' + AND o_orderDATE >= DATE '1994-01-01' + AND o_orderDATE < DATE '1994-01-01' + INTERVAL '1' YEAR +GROUP BY + n_name +ORDER BY + revenue DESC; +---- +sort + ├── columns: n_name:42(string!null) revenue:49(float) + ├── key: (42) + ├── fd: (42)-->(49) + ├── ordering: -49 + └── group-by + ├── columns: n_name:42(string!null) sum:49(float) + ├── grouping columns: n_name:42(string!null) + ├── key: (42) + ├── fd: (42)-->(49) + ├── project + │ ├── columns: column48:48(float) n_name:42(string!null) + │ ├── inner-join + │ │ ├── columns: c_custkey:1(int!null) c_nationkey:4(int!null) o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_suppkey:20(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_name:42(string!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ ├── fd: ()-->(46), (1)-->(4), (9)-->(10,13), (34)-->(37), (41)-->(42,43), (43)==(45), (45)==(43), (37)==(4,41), (41)==(4,37), (20)==(34), (34)==(20), (9)==(18), (18)==(9), (1)==(10), (10)==(1), (4)==(37,41) + │ │ ├── inner-join + │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_suppkey:20(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_name:42(string!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ ├── fd: ()-->(46), (9)-->(10,13), (34)-->(37), (41)-->(42,43), (43)==(45), (45)==(43), (37)==(41), (41)==(37), (20)==(34), (34)==(20), (9)==(18), (18)==(9) + │ │ │ ├── inner-join + │ │ │ │ ├── columns: l_orderkey:18(int!null) l_suppkey:20(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_name:42(string!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ ├── fd: ()-->(46), (34)-->(37), (41)-->(42,43), (43)==(45), (45)==(43), (37)==(41), (41)==(37), (20)==(34), (34)==(20) + │ │ │ │ ├── scan lineitem + │ │ │ │ │ └── columns: l_orderkey:18(int!null) l_suppkey:20(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) + │ │ │ │ ├── inner-join + │ │ │ │ │ ├── columns: s_suppkey:34(int!null) s_nationkey:37(int!null) n_nationkey:41(int!null) n_name:42(string!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ ├── key: (34) + │ │ │ │ │ ├── fd: ()-->(46), (34)-->(37), (41)-->(42,43), (43)==(45), (45)==(43), (37)==(41), (41)==(37) + │ │ │ │ │ ├── scan supplier@s_nk + │ │ │ │ │ │ ├── columns: s_suppkey:34(int!null) s_nationkey:37(int!null) + │ │ │ │ │ │ ├── key: (34) + │ │ │ │ │ │ └── fd: (34)-->(37) + │ │ │ │ │ ├── inner-join (lookup nation) + │ │ │ │ │ │ ├── columns: n_nationkey:41(int!null) n_name:42(string!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ │ ├── key columns: [41] = [41] + │ │ │ │ │ │ ├── key: (41) + │ │ │ │ │ │ ├── fd: ()-->(46), (41)-->(42,43), (43)==(45), (45)==(43) + │ │ │ │ │ │ ├── inner-join (lookup nation@n_rk) + │ │ │ │ │ │ │ ├── columns: n_nationkey:41(int!null) n_regionkey:43(int!null) r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ │ │ ├── key columns: [45] = [43] + │ │ │ │ │ │ │ ├── key: (41) + │ │ │ │ │ │ │ ├── fd: ()-->(46), (41)-->(43), (43)==(45), (45)==(43) + │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ │ │ │ ├── key: (45) + │ │ │ │ │ │ │ │ ├── fd: ()-->(46) + │ │ │ │ │ │ │ │ ├── scan region + │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:45(int!null) r_name:46(string!null) + │ │ │ │ │ │ │ │ │ ├── key: (45) + │ │ │ │ │ │ │ │ │ └── fd: (45)-->(46) + │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ └── r_name = 'ASIA' [type=bool, outer=(46), constraints=(/46: [/'ASIA' - /'ASIA']; tight), fd=()-->(46)] + │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ └── filters + │ │ │ │ │ └── s_nationkey = n_nationkey [type=bool, outer=(37,41), constraints=(/37: (/NULL - ]; /41: (/NULL - ]), fd=(37)==(41), (41)==(37)] + │ │ │ │ └── filters + │ │ │ │ └── l_suppkey = s_suppkey [type=bool, outer=(20,34), constraints=(/20: (/NULL - ]; /34: (/NULL - ]), fd=(20)==(34), (34)==(20)] + │ │ │ ├── index-join orders + │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) + │ │ │ │ ├── key: (9) + │ │ │ │ ├── fd: (9)-->(10,13) + │ │ │ │ └── scan orders@o_od + │ │ │ │ ├── columns: o_orderkey:9(int!null) o_orderdate:13(date!null) + │ │ │ │ ├── constraint: /13/9: [/'1994-01-01' - /'1994-12-31'] + │ │ │ │ ├── key: (9) + │ │ │ │ └── fd: (9)-->(13) + │ │ │ └── filters + │ │ │ └── l_orderkey = o_orderkey [type=bool, outer=(9,18), constraints=(/9: (/NULL - ]; /18: (/NULL - ]), fd=(9)==(18), (18)==(9)] + │ │ ├── scan customer@c_nk + │ │ │ ├── columns: c_custkey:1(int!null) c_nationkey:4(int!null) + │ │ │ ├── key: (1) + │ │ │ └── fd: (1)-->(4) + │ │ └── filters + │ │ ├── c_custkey = o_custkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ └── c_nationkey = s_nationkey [type=bool, outer=(4,37), constraints=(/4: (/NULL - ]; /37: (/NULL - ]), fd=(4)==(37), (37)==(4)] + │ └── projections + │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(23,24)] + └── aggregations + └── sum [type=float, outer=(48)] + └── variable: column48 [type=float] + +# -------------------------------------------------- +# Q6 +# Forecasting Revenue Change +# Quantifies the amount of revenue increase that would have resulted from +# eliminating certain companywide discounts in a given percentage range in a +# given year. Asking this type of "what if" query can be used to look for ways +# to increase revenues. +# +# Considers all the lineitems shipped in a given year with discounts between +# DISCOUNT-0.01 and DISCOUNT+0.01. The query lists the amount by which the total +# revenue would have increased if these discounts had been eliminated for +# lineitems with l_quantity less than quantity. Note that the potential revenue +# increase is equal to the sum of [l_extendedprice * l_discount] for all +# lineitems with discounts and quantities in the qualifying range. +# -------------------------------------------------- +opt +SELECT + sum(l_extendedprice * l_discount) AS revenue +FROM + lineitem +WHERE + l_shipdate >= DATE '1994-01-01' + AND l_shipdate < DATE '1994-01-01' + INTERVAL '1' YEAR + AND l_discount BETWEEN 0.06 - 0.01 AND 0.06 + 0.01 + AND l_quantity < 24; +---- +scalar-group-by + ├── columns: revenue:18(float) + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(18) + ├── project + │ ├── columns: column17:17(float) + │ ├── select + │ │ ├── columns: l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipdate:11(date!null) + │ │ ├── index-join lineitem + │ │ │ ├── columns: l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipdate:11(date!null) + │ │ │ └── scan lineitem@l_sd + │ │ │ ├── columns: l_orderkey:1(int!null) l_linenumber:4(int!null) l_shipdate:11(date!null) + │ │ │ ├── constraint: /11/1/4: [/'1994-01-01' - /'1994-12-31'] + │ │ │ ├── key: (1,4) + │ │ │ └── fd: (1,4)-->(11) + │ │ └── filters + │ │ ├── l_discount >= 0.05 [type=bool, outer=(7), constraints=(/7: [/0.05 - ]; tight)] + │ │ ├── l_discount <= 0.07 [type=bool, outer=(7), constraints=(/7: (/NULL - /0.07]; tight)] + │ │ └── l_quantity < 24.0 [type=bool, outer=(5), constraints=(/5: (/NULL - /23.999999999999996]; tight)] + │ └── projections + │ └── l_extendedprice * l_discount [type=float, outer=(6,7)] + └── aggregations + └── sum [type=float, outer=(17)] + └── variable: column17 [type=float] + +# -------------------------------------------------- +# Q7 +# Volume Shipping +# Determines the value of goods shipped between certain nations to help in the +# re-negotiation of shipping contracts. +# +# Finds, for two given nations, the gross discounted revenues derived from +# lineitems in which parts were shipped from a supplier in either nation to a +# customer in the other nation during 1995 and 1996. The query lists the +# supplier nation, the customer nation, the year, and the revenue from shipments +# that took place in that year. The query orders the answer by Supplier nation, +# Customer nation, and year (all ascending). +# +# TODO: +# 1. Join ordering +# -------------------------------------------------- +opt +SELECT + supp_nation, + cust_nation, + l_year, sum(volume) AS revenue +FROM ( + SELECT + n1.n_name AS supp_nation, + n2.n_name AS cust_nation, + extract(year FROM l_shipdate) AS l_year, + l_extendedprice * (1 - l_discount) AS volume + FROM + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2 + WHERE + s_suppkey = l_suppkey + AND o_orderkey = l_orderkey + AND c_custkey = o_custkey + AND s_nationkey = n1.n_nationkey + AND c_nationkey = n2.n_nationkey + AND ( + (n1.n_name = 'FRANCE' AND n2.n_name = 'GERMANY') + or (n1.n_name = 'GERMANY' AND n2.n_name = 'FRANCE') + ) + AND l_shipdate BETWEEN DATE '1995-01-01' AND DATE '1996-12-31' + ) AS shipping +GROUP BY + supp_nation, + cust_nation, + l_year +ORDER BY + supp_nation, + cust_nation, + l_year; +---- +group-by + ├── columns: supp_nation:42(string!null) cust_nation:46(string!null) l_year:49(int) revenue:51(float) + ├── grouping columns: n1.n_name:42(string!null) n2.n_name:46(string!null) l_year:49(int) + ├── key: (42,46,49) + ├── fd: (42,46,49)-->(51) + ├── ordering: +42,+46,+49 + ├── sort + │ ├── columns: n1.n_name:42(string!null) n2.n_name:46(string!null) l_year:49(int) volume:50(float) + │ ├── ordering: +42,+46,+49 + │ └── project + │ ├── columns: l_year:49(int) volume:50(float) n1.n_name:42(string!null) n2.n_name:46(string!null) + │ ├── inner-join + │ │ ├── columns: s_suppkey:1(int!null) s_nationkey:4(int!null) l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ ├── fd: (1)-->(4), (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25), (8)==(24), (24)==(8), (1)==(10), (10)==(1), (4)==(41), (41)==(4) + │ │ ├── inner-join + │ │ │ ├── columns: l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ ├── fd: (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25), (8)==(24), (24)==(8) + │ │ │ ├── inner-join + │ │ │ │ ├── columns: o_orderkey:24(int!null) o_custkey:25(int!null) c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ ├── key: (24,41) + │ │ │ │ ├── fd: (24)-->(25), (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36), (25)==(33), (33)==(25) + │ │ │ │ ├── inner-join + │ │ │ │ │ ├── columns: c_custkey:33(int!null) c_nationkey:36(int!null) n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ │ ├── key: (33,41) + │ │ │ │ │ ├── fd: (33)-->(36), (41)-->(42), (45)-->(46), (36)==(45), (45)==(36) + │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ ├── columns: n1.n_nationkey:41(int!null) n1.n_name:42(string!null) n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ │ │ ├── key: (41,45) + │ │ │ │ │ │ ├── fd: (41)-->(42), (45)-->(46) + │ │ │ │ │ │ ├── scan n1 + │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:41(int!null) n1.n_name:42(string!null) + │ │ │ │ │ │ │ ├── key: (41) + │ │ │ │ │ │ │ └── fd: (41)-->(42) + │ │ │ │ │ │ ├── scan n2 + │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:45(int!null) n2.n_name:46(string!null) + │ │ │ │ │ │ │ ├── key: (45) + │ │ │ │ │ │ │ └── fd: (45)-->(46) + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── ((n1.n_name = 'FRANCE') AND (n2.n_name = 'GERMANY')) OR ((n1.n_name = 'GERMANY') AND (n2.n_name = 'FRANCE')) [type=bool, outer=(42,46)] + │ │ │ │ │ ├── scan customer@c_nk + │ │ │ │ │ │ ├── columns: c_custkey:33(int!null) c_nationkey:36(int!null) + │ │ │ │ │ │ ├── key: (33) + │ │ │ │ │ │ └── fd: (33)-->(36) + │ │ │ │ │ └── filters + │ │ │ │ │ └── c_nationkey = n2.n_nationkey [type=bool, outer=(36,45), constraints=(/36: (/NULL - ]; /45: (/NULL - ]), fd=(36)==(45), (45)==(36)] + │ │ │ │ ├── scan orders@o_ck + │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_custkey:25(int!null) + │ │ │ │ │ ├── key: (24) + │ │ │ │ │ └── fd: (24)-->(25) + │ │ │ │ └── filters + │ │ │ │ └── c_custkey = o_custkey [type=bool, outer=(25,33), constraints=(/25: (/NULL - ]; /33: (/NULL - ]), fd=(25)==(33), (33)==(25)] + │ │ │ ├── index-join lineitem + │ │ │ │ ├── columns: l_orderkey:8(int!null) l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) + │ │ │ │ └── scan lineitem@l_sd + │ │ │ │ ├── columns: l_orderkey:8(int!null) l_linenumber:11(int!null) l_shipdate:18(date!null) + │ │ │ │ ├── constraint: /18/8/11: [/'1995-01-01' - /'1996-12-31'] + │ │ │ │ ├── key: (8,11) + │ │ │ │ └── fd: (8,11)-->(18) + │ │ │ └── filters + │ │ │ └── o_orderkey = l_orderkey [type=bool, outer=(8,24), constraints=(/8: (/NULL - ]; /24: (/NULL - ]), fd=(8)==(24), (24)==(8)] + │ │ ├── scan supplier@s_nk + │ │ │ ├── columns: s_suppkey:1(int!null) s_nationkey:4(int!null) + │ │ │ ├── key: (1) + │ │ │ └── fd: (1)-->(4) + │ │ └── filters + │ │ ├── s_suppkey = l_suppkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ └── s_nationkey = n1.n_nationkey [type=bool, outer=(4,41), constraints=(/4: (/NULL - ]; /41: (/NULL - ]), fd=(4)==(41), (41)==(4)] + │ └── projections + │ ├── extract('year', l_shipdate) [type=int, outer=(18)] + │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(13,14)] + └── aggregations + └── sum [type=float, outer=(50)] + └── variable: volume [type=float] + +# -------------------------------------------------- +# Q8 +# National Market Share +# Determines how the market share of a given nation within a given region has +# changed over two years for a given part type. +# +# The market share for a given nation within a given region is defined as the +# fraction of the revenue, the sum of [l_extendedprice * (1-l_discount)], from +# the products of a specified type in that region that was supplied by suppliers +# from the given nation. The query determines this for the years 1995 and 1996 +# presented in this order. +# +# TODO: +# 1. Join ordering +# 2. Push down equivalent column comparisons +# -------------------------------------------------- +opt +SELECT + o_year, + sum(CASE + WHEN nation = 'BRAZIL' + THEN volume + ELSE 0 + END) / sum(volume) AS mkt_share +FROM ( + SELECT + extract(year FROM o_orderdate) AS o_year, + l_extendedprice * (1 - l_discount) AS volume, + n2.n_name AS nation + FROM + part, + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2, + region + WHERE + p_partkey = l_partkey + AND s_suppkey = l_suppkey + AND l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND c_nationkey = n1.n_nationkey + AND n1.n_regionkey = r_regionkey + AND r_name = 'AMERICA' + AND s_nationkey = n2.n_nationkey + AND o_orderdate BETWEEN DATE '1995-01-01' AND DATE '1996-12-31' + AND p_type = 'ECONOMY ANODIZED STEEL' + ) AS all_nations +GROUP BY + o_year +ORDER BY + o_year; +---- +sort + ├── columns: o_year:61(int) mkt_share:66(float) + ├── side-effects + ├── key: (61) + ├── fd: (61)-->(66) + ├── ordering: +61 + └── project + ├── columns: mkt_share:66(float) o_year:61(int) + ├── side-effects + ├── key: (61) + ├── fd: (61)-->(66) + ├── group-by + │ ├── columns: o_year:61(int) sum:64(float) sum:65(float) + │ ├── grouping columns: o_year:61(int) + │ ├── key: (61) + │ ├── fd: (61)-->(64,65) + │ ├── project + │ │ ├── columns: column63:63(float) o_year:61(int) volume:62(float) + │ │ ├── project + │ │ │ ├── columns: o_year:61(int) volume:62(float) n2.n_name:55(string!null) + │ │ │ ├── inner-join (lookup part) + │ │ │ │ ├── columns: p_partkey:1(int!null) p_type:5(string!null) s_suppkey:10(int!null) s_nationkey:13(int!null) l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ ├── key columns: [18] = [1] + │ │ │ │ ├── fd: ()-->(5,59), (10)-->(13), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17), (10)==(19), (19)==(10), (13)==(54), (54)==(13), (1)==(18), (18)==(1) + │ │ │ │ ├── inner-join + │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_nationkey:13(int!null) l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ ├── fd: ()-->(59), (10)-->(13), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17), (10)==(19), (19)==(10), (13)==(54), (54)==(13) + │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ ├── columns: l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ ├── fd: ()-->(59), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34), (17)==(33), (33)==(17) + │ │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ ├── key: (33,54) + │ │ │ │ │ │ │ ├── fd: ()-->(59), (33)-->(34,37), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45), (34)==(42), (42)==(34) + │ │ │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ │ │ ├── columns: c_custkey:42(int!null) c_nationkey:45(int!null) n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ ├── key: (42,54) + │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (42)-->(45), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52), (45)==(50), (50)==(45) + │ │ │ │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ ├── key: (50,54) + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (50)-->(52), (54)-->(55), (52)==(58), (58)==(52) + │ │ │ │ │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:54(int!null) n2.n_name:55(string!null) r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ │ ├── key: (54,58) + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59), (54)-->(55) + │ │ │ │ │ │ │ │ │ │ ├── scan n2 + │ │ │ │ │ │ │ │ │ │ │ ├── columns: n2.n_nationkey:54(int!null) n2.n_name:55(string!null) + │ │ │ │ │ │ │ │ │ │ │ ├── key: (54) + │ │ │ │ │ │ │ │ │ │ │ └── fd: (54)-->(55) + │ │ │ │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ │ │ ├── key: (58) + │ │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(59) + │ │ │ │ │ │ │ │ │ │ │ ├── scan region + │ │ │ │ │ │ │ │ │ │ │ │ ├── columns: r_regionkey:58(int!null) r_name:59(string!null) + │ │ │ │ │ │ │ │ │ │ │ │ ├── key: (58) + │ │ │ │ │ │ │ │ │ │ │ │ └── fd: (58)-->(59) + │ │ │ │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ │ │ │ └── r_name = 'AMERICA' [type=bool, outer=(59), constraints=(/59: [/'AMERICA' - /'AMERICA']; tight), fd=()-->(59)] + │ │ │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ │ ├── scan n1@n_rk + │ │ │ │ │ │ │ │ │ │ ├── columns: n1.n_nationkey:50(int!null) n1.n_regionkey:52(int!null) + │ │ │ │ │ │ │ │ │ │ ├── key: (50) + │ │ │ │ │ │ │ │ │ │ └── fd: (50)-->(52) + │ │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ │ └── n1.n_regionkey = r_regionkey [type=bool, outer=(52,58), constraints=(/52: (/NULL - ]; /58: (/NULL - ]), fd=(52)==(58), (58)==(52)] + │ │ │ │ │ │ │ │ ├── scan customer@c_nk + │ │ │ │ │ │ │ │ │ ├── columns: c_custkey:42(int!null) c_nationkey:45(int!null) + │ │ │ │ │ │ │ │ │ ├── key: (42) + │ │ │ │ │ │ │ │ │ └── fd: (42)-->(45) + │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ └── c_nationkey = n1.n_nationkey [type=bool, outer=(45,50), constraints=(/45: (/NULL - ]; /50: (/NULL - ]), fd=(45)==(50), (50)==(45)] + │ │ │ │ │ │ │ ├── index-join orders + │ │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_custkey:34(int!null) o_orderdate:37(date!null) + │ │ │ │ │ │ │ │ ├── key: (33) + │ │ │ │ │ │ │ │ ├── fd: (33)-->(34,37) + │ │ │ │ │ │ │ │ └── scan orders@o_od + │ │ │ │ │ │ │ │ ├── columns: o_orderkey:33(int!null) o_orderdate:37(date!null) + │ │ │ │ │ │ │ │ ├── constraint: /37/33: [/'1995-01-01' - /'1996-12-31'] + │ │ │ │ │ │ │ │ ├── key: (33) + │ │ │ │ │ │ │ │ └── fd: (33)-->(37) + │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ └── o_custkey = c_custkey [type=bool, outer=(34,42), constraints=(/34: (/NULL - ]; /42: (/NULL - ]), fd=(34)==(42), (42)==(34)] + │ │ │ │ │ │ ├── scan lineitem + │ │ │ │ │ │ │ └── columns: l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_extendedprice:22(float!null) l_discount:23(float!null) + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── l_orderkey = o_orderkey [type=bool, outer=(17,33), constraints=(/17: (/NULL - ]; /33: (/NULL - ]), fd=(17)==(33), (33)==(17)] + │ │ │ │ │ ├── scan supplier@s_nk + │ │ │ │ │ │ ├── columns: s_suppkey:10(int!null) s_nationkey:13(int!null) + │ │ │ │ │ │ ├── key: (10) + │ │ │ │ │ │ └── fd: (10)-->(13) + │ │ │ │ │ └── filters + │ │ │ │ │ ├── s_suppkey = l_suppkey [type=bool, outer=(10,19), constraints=(/10: (/NULL - ]; /19: (/NULL - ]), fd=(10)==(19), (19)==(10)] + │ │ │ │ │ └── s_nationkey = n2.n_nationkey [type=bool, outer=(13,54), constraints=(/13: (/NULL - ]; /54: (/NULL - ]), fd=(13)==(54), (54)==(13)] + │ │ │ │ └── filters + │ │ │ │ └── p_type = 'ECONOMY ANODIZED STEEL' [type=bool, outer=(5), constraints=(/5: [/'ECONOMY ANODIZED STEEL' - /'ECONOMY ANODIZED STEEL']; tight), fd=()-->(5)] + │ │ │ └── projections + │ │ │ ├── extract('year', o_orderdate) [type=int, outer=(37)] + │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(22,23)] + │ │ └── projections + │ │ └── CASE WHEN n2.n_name = 'BRAZIL' THEN volume ELSE 0.0 END [type=float, outer=(55,62)] + │ └── aggregations + │ ├── sum [type=float, outer=(63)] + │ │ └── variable: column63 [type=float] + │ └── sum [type=float, outer=(62)] + │ └── variable: volume [type=float] + └── projections + └── sum / sum [type=float, outer=(64,65), side-effects] + +# -------------------------------------------------- +# Q9 +# Product Type Profit Measure +# Determines how much profit is made on a given line of parts, broken out by +# supplier nation and year. +# +# Finds, for each nation and each year, the profit for all parts ordered in that +# year that contain a specified substring in their names and that were filled by +# a supplier in that nation. The profit is defined as the sum of: +# +# [(l_extendedprice*(1-l_discount)) - (ps_supplycost * l_quantity)] +# +# for all lineitems describing parts in the specified line. The query lists the +# nations in ascending alphabetical order and, for each nation, the year and +# profit in descending order by year (most recent first). +# +# TODO: +# 1. Join ordering +# 2. Push down equivalent column comparisons +# -------------------------------------------------- +opt +SELECT + nation, + o_year, + sum(amount) AS sum_profit +FROM ( + SELECT + n_name AS nation, + extract(year FROM o_orderdate) AS o_year, + l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity AS amount + FROM + part, + supplier, + lineitem, + partsupp, + orders, + nation + WHERE + s_suppkey = l_suppkey + AND ps_suppkey = l_suppkey + AND ps_partkey = l_partkey + AND p_partkey = l_partkey + AND o_orderkey = l_orderkey + AND s_nationkey = n_nationkey + AND p_name LIKE '%green%' + ) AS profit +GROUP BY + nation, + o_year +ORDER BY + nation, + o_year DESC; +---- +sort + ├── columns: nation:48(string!null) o_year:51(int) sum_profit:53(float) + ├── key: (48,51) + ├── fd: (48,51)-->(53) + ├── ordering: +48,-51 + └── group-by + ├── columns: n_name:48(string!null) o_year:51(int) sum:53(float) + ├── grouping columns: n_name:48(string!null) o_year:51(int) + ├── key: (48,51) + ├── fd: (48,51)-->(53) + ├── project + │ ├── columns: o_year:51(int) amount:52(float) n_name:48(string!null) + │ ├── inner-join (lookup part) + │ │ ├── columns: p_partkey:1(int!null) p_name:2(string!null) s_suppkey:10(int!null) s_nationkey:13(int!null) l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_quantity:21(float!null) l_extendedprice:22(float!null) l_discount:23(float!null) ps_partkey:33(int!null) ps_suppkey:34(int!null) ps_supplycost:36(float!null) o_orderkey:38(int!null) o_orderdate:42(date!null) n_nationkey:47(int!null) n_name:48(string!null) + │ │ ├── key columns: [18] = [1] + │ │ ├── fd: (1)-->(2), (10)-->(13), (33,34)-->(36), (38)-->(42), (47)-->(48), (19)==(10,34), (34)==(10,19), (18)==(1,33), (33)==(1,18), (17)==(38), (38)==(17), (10)==(19,34), (13)==(47), (47)==(13), (1)==(18,33) + │ │ ├── inner-join + │ │ │ ├── columns: s_suppkey:10(int!null) s_nationkey:13(int!null) l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_quantity:21(float!null) l_extendedprice:22(float!null) l_discount:23(float!null) ps_partkey:33(int!null) ps_suppkey:34(int!null) ps_supplycost:36(float!null) o_orderkey:38(int!null) o_orderdate:42(date!null) n_nationkey:47(int!null) n_name:48(string!null) + │ │ │ ├── fd: (10)-->(13), (33,34)-->(36), (38)-->(42), (47)-->(48), (19)==(10,34), (34)==(10,19), (18)==(33), (33)==(18), (17)==(38), (38)==(17), (10)==(19,34), (13)==(47), (47)==(13) + │ │ │ ├── inner-join + │ │ │ │ ├── columns: l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_quantity:21(float!null) l_extendedprice:22(float!null) l_discount:23(float!null) ps_partkey:33(int!null) ps_suppkey:34(int!null) ps_supplycost:36(float!null) o_orderkey:38(int!null) o_orderdate:42(date!null) n_nationkey:47(int!null) n_name:48(string!null) + │ │ │ │ ├── fd: (33,34)-->(36), (38)-->(42), (47)-->(48), (19)==(34), (34)==(19), (18)==(33), (33)==(18), (17)==(38), (38)==(17) + │ │ │ │ ├── inner-join + │ │ │ │ │ ├── columns: ps_partkey:33(int!null) ps_suppkey:34(int!null) ps_supplycost:36(float!null) o_orderkey:38(int!null) o_orderdate:42(date!null) n_nationkey:47(int!null) n_name:48(string!null) + │ │ │ │ │ ├── key: (33,34,38,47) + │ │ │ │ │ ├── fd: (33,34)-->(36), (38)-->(42), (47)-->(48) + │ │ │ │ │ ├── inner-join + │ │ │ │ │ │ ├── columns: o_orderkey:38(int!null) o_orderdate:42(date!null) n_nationkey:47(int!null) n_name:48(string!null) + │ │ │ │ │ │ ├── key: (38,47) + │ │ │ │ │ │ ├── fd: (38)-->(42), (47)-->(48) + │ │ │ │ │ │ ├── scan orders@o_od + │ │ │ │ │ │ │ ├── columns: o_orderkey:38(int!null) o_orderdate:42(date!null) + │ │ │ │ │ │ │ ├── key: (38) + │ │ │ │ │ │ │ └── fd: (38)-->(42) + │ │ │ │ │ │ ├── scan nation + │ │ │ │ │ │ │ ├── columns: n_nationkey:47(int!null) n_name:48(string!null) + │ │ │ │ │ │ │ ├── key: (47) + │ │ │ │ │ │ │ └── fd: (47)-->(48) + │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ ├── scan partsupp + │ │ │ │ │ │ ├── columns: ps_partkey:33(int!null) ps_suppkey:34(int!null) ps_supplycost:36(float!null) + │ │ │ │ │ │ ├── key: (33,34) + │ │ │ │ │ │ └── fd: (33,34)-->(36) + │ │ │ │ │ └── filters (true) + │ │ │ │ ├── scan lineitem + │ │ │ │ │ └── columns: l_orderkey:17(int!null) l_partkey:18(int!null) l_suppkey:19(int!null) l_quantity:21(float!null) l_extendedprice:22(float!null) l_discount:23(float!null) + │ │ │ │ └── filters + │ │ │ │ ├── ps_suppkey = l_suppkey [type=bool, outer=(19,34), constraints=(/19: (/NULL - ]; /34: (/NULL - ]), fd=(19)==(34), (34)==(19)] + │ │ │ │ ├── ps_partkey = l_partkey [type=bool, outer=(18,33), constraints=(/18: (/NULL - ]; /33: (/NULL - ]), fd=(18)==(33), (33)==(18)] + │ │ │ │ └── o_orderkey = l_orderkey [type=bool, outer=(17,38), constraints=(/17: (/NULL - ]; /38: (/NULL - ]), fd=(17)==(38), (38)==(17)] + │ │ │ ├── scan supplier@s_nk + │ │ │ │ ├── columns: s_suppkey:10(int!null) s_nationkey:13(int!null) + │ │ │ │ ├── key: (10) + │ │ │ │ └── fd: (10)-->(13) + │ │ │ └── filters + │ │ │ ├── s_suppkey = l_suppkey [type=bool, outer=(10,19), constraints=(/10: (/NULL - ]; /19: (/NULL - ]), fd=(10)==(19), (19)==(10)] + │ │ │ └── s_nationkey = n_nationkey [type=bool, outer=(13,47), constraints=(/13: (/NULL - ]; /47: (/NULL - ]), fd=(13)==(47), (47)==(13)] + │ │ └── filters + │ │ └── p_name LIKE '%green%' [type=bool, outer=(2), constraints=(/2: (/NULL - ])] + │ └── projections + │ ├── extract('year', o_orderdate) [type=int, outer=(42)] + │ └── (l_extendedprice * (1.0 - l_discount)) - (ps_supplycost * l_quantity) [type=float, outer=(21-23,36)] + └── aggregations + └── sum [type=float, outer=(52)] + └── variable: amount [type=float] + +# -------------------------------------------------- +# Q10 +# Returned Item Reporting +# Identifies customers who might be having problems with the parts that are +# shipped to them. +# +# Finds the top 20 customers, in terms of their effect on lost revenue for a +# given quarter, who have returned parts. The query considers only parts that +# were ordered in the specified quarter. The query lists the customer's name, +# address, nation, phone number, account balance, comment information and +# revenue lost. The customers are listed in descending order of lost revenue. +# Revenue lost is defined as sum(l_extendedprice*(1-l_discount)) for all +# qualifying lineitems. +# -------------------------------------------------- +opt +SELECT + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) AS revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment +FROM + customer, + orders, + lineitem, + nation +WHERE + c_custkey = o_custkey + AND l_orderkey = o_orderkey + AND o_orderDATE >= DATE '1993-10-01' + AND o_orderDATE < DATE '1993-10-01' + INTERVAL '3' MONTH + AND l_returnflag = 'R' + AND c_nationkey = n_nationkey +GROUP BY + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment +ORDER BY + revenue DESC +LIMIT 20; +---- +limit + ├── columns: c_custkey:1(int!null) c_name:2(string) revenue:39(float) c_acctbal:6(float) n_name:35(string) c_address:3(string) c_phone:5(string) c_comment:8(string) + ├── internal-ordering: -39 + ├── cardinality: [0 - 20] + ├── key: (1) + ├── fd: (1)-->(2,3,5,6,8,35,39) + ├── ordering: -39 + ├── sort + │ ├── columns: c_custkey:1(int!null) c_name:2(string) c_address:3(string) c_phone:5(string) c_acctbal:6(float) c_comment:8(string) n_name:35(string) sum:39(float) + │ ├── key: (1) + │ ├── fd: (1)-->(2,3,5,6,8,35,39) + │ ├── ordering: -39 + │ └── group-by + │ ├── columns: c_custkey:1(int!null) c_name:2(string) c_address:3(string) c_phone:5(string) c_acctbal:6(float) c_comment:8(string) n_name:35(string) sum:39(float) + │ ├── grouping columns: c_custkey:1(int!null) + │ ├── key: (1) + │ ├── fd: (1)-->(2,3,5,6,8,35,39) + │ ├── project + │ │ ├── columns: column38:38(float) c_custkey:1(int!null) c_name:2(string!null) c_address:3(string!null) c_phone:5(string!null) c_acctbal:6(float!null) c_comment:8(string!null) n_name:35(string!null) + │ │ ├── fd: (1)-->(2,3,5,6,8,35) + │ │ ├── inner-join + │ │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) c_address:3(string!null) c_nationkey:4(int!null) c_phone:5(string!null) c_acctbal:6(float!null) c_comment:8(string!null) o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_returnflag:26(string!null) n_nationkey:34(int!null) n_name:35(string!null) + │ │ │ ├── fd: ()-->(26), (1)-->(2-6,8), (9)-->(10,13), (34)-->(35), (9)==(18), (18)==(9), (1)==(10), (10)==(1), (4)==(34), (34)==(4) + │ │ │ ├── inner-join + │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_returnflag:26(string!null) n_nationkey:34(int!null) n_name:35(string!null) + │ │ │ │ ├── fd: ()-->(26), (9)-->(10,13), (34)-->(35), (9)==(18), (18)==(9) + │ │ │ │ ├── inner-join + │ │ │ │ │ ├── columns: l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_returnflag:26(string!null) n_nationkey:34(int!null) n_name:35(string!null) + │ │ │ │ │ ├── fd: ()-->(26), (34)-->(35) + │ │ │ │ │ ├── scan nation + │ │ │ │ │ │ ├── columns: n_nationkey:34(int!null) n_name:35(string!null) + │ │ │ │ │ │ ├── key: (34) + │ │ │ │ │ │ └── fd: (34)-->(35) + │ │ │ │ │ ├── select + │ │ │ │ │ │ ├── columns: l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_returnflag:26(string!null) + │ │ │ │ │ │ ├── fd: ()-->(26) + │ │ │ │ │ │ ├── scan lineitem + │ │ │ │ │ │ │ └── columns: l_orderkey:18(int!null) l_extendedprice:23(float!null) l_discount:24(float!null) l_returnflag:26(string!null) + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── l_returnflag = 'R' [type=bool, outer=(26), constraints=(/26: [/'R' - /'R']; tight), fd=()-->(26)] + │ │ │ │ │ └── filters (true) + │ │ │ │ ├── index-join orders + │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_orderdate:13(date!null) + │ │ │ │ │ ├── key: (9) + │ │ │ │ │ ├── fd: (9)-->(10,13) + │ │ │ │ │ └── scan orders@o_od + │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_orderdate:13(date!null) + │ │ │ │ │ ├── constraint: /13/9: [/'1993-10-01' - /'1993-12-31'] + │ │ │ │ │ ├── key: (9) + │ │ │ │ │ └── fd: (9)-->(13) + │ │ │ │ └── filters + │ │ │ │ └── l_orderkey = o_orderkey [type=bool, outer=(9,18), constraints=(/9: (/NULL - ]; /18: (/NULL - ]), fd=(9)==(18), (18)==(9)] + │ │ │ ├── scan customer + │ │ │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) c_address:3(string!null) c_nationkey:4(int!null) c_phone:5(string!null) c_acctbal:6(float!null) c_comment:8(string!null) + │ │ │ │ ├── key: (1) + │ │ │ │ └── fd: (1)-->(2-6,8) + │ │ │ └── filters + │ │ │ ├── c_custkey = o_custkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ │ └── c_nationkey = n_nationkey [type=bool, outer=(4,34), constraints=(/4: (/NULL - ]; /34: (/NULL - ]), fd=(4)==(34), (34)==(4)] + │ │ └── projections + │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(23,24)] + │ └── aggregations + │ ├── sum [type=float, outer=(38)] + │ │ └── variable: column38 [type=float] + │ ├── const-agg [type=string, outer=(2)] + │ │ └── variable: c_name [type=string] + │ ├── const-agg [type=string, outer=(3)] + │ │ └── variable: c_address [type=string] + │ ├── const-agg [type=string, outer=(5)] + │ │ └── variable: c_phone [type=string] + │ ├── const-agg [type=float, outer=(6)] + │ │ └── variable: c_acctbal [type=float] + │ ├── const-agg [type=string, outer=(8)] + │ │ └── variable: c_comment [type=string] + │ └── const-agg [type=string, outer=(35)] + │ └── variable: n_name [type=string] + └── const: 20 [type=int] + +# -------------------------------------------------- +# Q11 +# Important Stock Identification +# Finds the most important subset of suppliers' stock in a given nation. +# +# Finds, from scanning the available stock of suppliers in a given nation, all +# the parts that represent a significant percentage of the total value of all +# available parts. The query displays the part number and the value of those +# parts in descending order of value. +# -------------------------------------------------- +opt +SELECT + ps_partkey, + sum(ps_supplycost * ps_availqty::float) AS value +FROM + partsupp, + supplier, + nation +WHERE + ps_suppkey = s_suppkey + AND s_nationkey = n_nationkey + AND n_name = 'GERMANY' +GROUP BY + ps_partkey HAVING + sum(ps_supplycost * ps_availqty::float) > ( + SELECT + sum(ps_supplycost * ps_availqty::float) * 0.0001 + FROM + partsupp, + supplier, + nation + WHERE + ps_suppkey = s_suppkey + AND s_nationkey = n_nationkey + AND n_name = 'GERMANY' + ) +ORDER BY + value DESC; +---- +sort + ├── columns: ps_partkey:1(int!null) value:18(float!null) + ├── key: (1) + ├── fd: (1)-->(18) + ├── ordering: -18 + └── select + ├── columns: ps_partkey:1(int!null) sum:18(float!null) + ├── key: (1) + ├── fd: (1)-->(18) + ├── group-by + │ ├── columns: ps_partkey:1(int!null) sum:18(float) + │ ├── grouping columns: ps_partkey:1(int!null) + │ ├── key: (1) + │ ├── fd: (1)-->(18) + │ ├── project + │ │ ├── columns: column17:17(float) ps_partkey:1(int!null) + │ │ ├── inner-join + │ │ │ ├── columns: ps_partkey:1(int!null) ps_suppkey:2(int!null) ps_availqty:3(int!null) ps_supplycost:4(float!null) s_suppkey:6(int!null) s_nationkey:9(int!null) n_nationkey:13(int!null) n_name:14(string!null) + │ │ │ ├── key: (1,6) + │ │ │ ├── fd: ()-->(14), (1,2)-->(3,4), (6)-->(9), (9)==(13), (13)==(9), (2)==(6), (6)==(2) + │ │ │ ├── scan partsupp + │ │ │ │ ├── columns: ps_partkey:1(int!null) ps_suppkey:2(int!null) ps_availqty:3(int!null) ps_supplycost:4(float!null) + │ │ │ │ ├── key: (1,2) + │ │ │ │ └── fd: (1,2)-->(3,4) + │ │ │ ├── inner-join (lookup supplier@s_nk) + │ │ │ │ ├── columns: s_suppkey:6(int!null) s_nationkey:9(int!null) n_nationkey:13(int!null) n_name:14(string!null) + │ │ │ │ ├── key columns: [13] = [9] + │ │ │ │ ├── key: (6) + │ │ │ │ ├── fd: ()-->(14), (6)-->(9), (9)==(13), (13)==(9) + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: n_nationkey:13(int!null) n_name:14(string!null) + │ │ │ │ │ ├── key: (13) + │ │ │ │ │ ├── fd: ()-->(14) + │ │ │ │ │ ├── scan nation + │ │ │ │ │ │ ├── columns: n_nationkey:13(int!null) n_name:14(string!null) + │ │ │ │ │ │ ├── key: (13) + │ │ │ │ │ │ └── fd: (13)-->(14) + │ │ │ │ │ └── filters + │ │ │ │ │ └── n_name = 'GERMANY' [type=bool, outer=(14), constraints=(/14: [/'GERMANY' - /'GERMANY']; tight), fd=()-->(14)] + │ │ │ │ └── filters (true) + │ │ │ └── filters + │ │ │ └── ps_suppkey = s_suppkey [type=bool, outer=(2,6), constraints=(/2: (/NULL - ]; /6: (/NULL - ]), fd=(2)==(6), (6)==(2)] + │ │ └── projections + │ │ └── ps_supplycost * ps_availqty::FLOAT8 [type=float, outer=(3,4)] + │ └── aggregations + │ └── sum [type=float, outer=(17)] + │ └── variable: column17 [type=float] + └── filters + └── gt [type=bool, outer=(18), constraints=(/18: (/NULL - ])] + ├── variable: sum [type=float] + └── subquery [type=float] + └── project + ├── columns: "?column?":37(float) + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(37) + ├── scalar-group-by + │ ├── columns: sum:36(float) + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(36) + │ ├── project + │ │ ├── columns: column35:35(float) + │ │ ├── inner-join + │ │ │ ├── columns: ps_suppkey:20(int!null) ps_availqty:21(int!null) ps_supplycost:22(float!null) s_suppkey:24(int!null) s_nationkey:27(int!null) n_nationkey:31(int!null) n_name:32(string!null) + │ │ │ ├── fd: ()-->(32), (24)-->(27), (27)==(31), (31)==(27), (20)==(24), (24)==(20) + │ │ │ ├── scan partsupp + │ │ │ │ └── columns: ps_suppkey:20(int!null) ps_availqty:21(int!null) ps_supplycost:22(float!null) + │ │ │ ├── inner-join (lookup supplier@s_nk) + │ │ │ │ ├── columns: s_suppkey:24(int!null) s_nationkey:27(int!null) n_nationkey:31(int!null) n_name:32(string!null) + │ │ │ │ ├── key columns: [31] = [27] + │ │ │ │ ├── key: (24) + │ │ │ │ ├── fd: ()-->(32), (24)-->(27), (27)==(31), (31)==(27) + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: n_nationkey:31(int!null) n_name:32(string!null) + │ │ │ │ │ ├── key: (31) + │ │ │ │ │ ├── fd: ()-->(32) + │ │ │ │ │ ├── scan nation + │ │ │ │ │ │ ├── columns: n_nationkey:31(int!null) n_name:32(string!null) + │ │ │ │ │ │ ├── key: (31) + │ │ │ │ │ │ └── fd: (31)-->(32) + │ │ │ │ │ └── filters + │ │ │ │ │ └── n_name = 'GERMANY' [type=bool, outer=(32), constraints=(/32: [/'GERMANY' - /'GERMANY']; tight), fd=()-->(32)] + │ │ │ │ └── filters (true) + │ │ │ └── filters + │ │ │ └── ps_suppkey = s_suppkey [type=bool, outer=(20,24), constraints=(/20: (/NULL - ]; /24: (/NULL - ]), fd=(20)==(24), (24)==(20)] + │ │ └── projections + │ │ └── ps_supplycost * ps_availqty::FLOAT8 [type=float, outer=(21,22)] + │ └── aggregations + │ └── sum [type=float, outer=(35)] + │ └── variable: column35 [type=float] + └── projections + └── sum * 0.0001 [type=float, outer=(36)] + +# -------------------------------------------------- +# Q12 +# Shipping Modes and Order Priority +# Determines whether selecting less expensive modes of shipping is negatively +# affecting the critical-priority orders by causing more parts to be received by +# customers after the committed date. +# +# Counts, by ship mode, for lineitems actually received by customers in a given +# year, the number of lineitems belonging to orders for which the l_receiptdate +# exceeds the l_commitdate for two different specified ship modes. Only +# lineitems that were actually shipped before the l_commitdate are considered. +# The late lineitems are partitioned into two groups, those with priority URGENT +# or HIGH, and those with a priority other than URGENT or HIGH. +# -------------------------------------------------- +opt +SELECT + l_shipmode, + sum(CASE + WHEN o_orderpriority = '1-URGENT' + OR o_orderpriority = '2-HIGH' + THEN 1 + ELSE 0 + END) AS high_line_count, + sum(CASE + WHEN o_orderpriority <> '1-URGENT' + AND o_orderpriority <> '2-HIGH' + THEN 1 + ELSE 0 + END) AS low_line_count +FROM + orders, + lineitem +WHERE + o_orderkey = l_orderkey + AND l_shipmode IN ('MAIL', 'SHIP') + AND l_commitdate < l_receiptdate + AND l_shipdate < l_commitdate + AND l_receiptdate >= DATE '1994-01-01' + AND l_receiptdate < DATE '1994-01-01' + INTERVAL '1' YEAR +GROUP BY + l_shipmode +ORDER BY + l_shipmode; +---- +group-by + ├── columns: l_shipmode:24(string!null) high_line_count:27(decimal) low_line_count:29(decimal) + ├── grouping columns: l_shipmode:24(string!null) + ├── key: (24) + ├── fd: (24)-->(27,29) + ├── ordering: +24 + ├── project + │ ├── columns: column26:26(int) column28:28(int) l_shipmode:24(string!null) + │ ├── ordering: +24 + │ ├── inner-join (lookup orders) + │ │ ├── columns: o_orderkey:1(int!null) o_orderpriority:6(string!null) l_orderkey:10(int!null) l_shipdate:20(date!null) l_commitdate:21(date!null) l_receiptdate:22(date!null) l_shipmode:24(string!null) + │ │ ├── key columns: [10] = [1] + │ │ ├── fd: (1)-->(6), (1)==(10), (10)==(1) + │ │ ├── ordering: +24 + │ │ ├── sort + │ │ │ ├── columns: l_orderkey:10(int!null) l_shipdate:20(date!null) l_commitdate:21(date!null) l_receiptdate:22(date!null) l_shipmode:24(string!null) + │ │ │ ├── ordering: +24 + │ │ │ └── select + │ │ │ ├── columns: l_orderkey:10(int!null) l_shipdate:20(date!null) l_commitdate:21(date!null) l_receiptdate:22(date!null) l_shipmode:24(string!null) + │ │ │ ├── index-join lineitem + │ │ │ │ ├── columns: l_orderkey:10(int!null) l_shipdate:20(date!null) l_commitdate:21(date!null) l_receiptdate:22(date!null) l_shipmode:24(string!null) + │ │ │ │ └── scan lineitem@l_rd + │ │ │ │ ├── columns: l_orderkey:10(int!null) l_linenumber:13(int!null) l_receiptdate:22(date!null) + │ │ │ │ ├── constraint: /22/10/13: [/'1994-01-01' - /'1994-12-31'] + │ │ │ │ ├── key: (10,13) + │ │ │ │ └── fd: (10,13)-->(22) + │ │ │ └── filters + │ │ │ ├── l_shipmode IN ('MAIL', 'SHIP') [type=bool, outer=(24), constraints=(/24: [/'MAIL' - /'MAIL'] [/'SHIP' - /'SHIP']; tight)] + │ │ │ ├── l_commitdate < l_receiptdate [type=bool, outer=(21,22), constraints=(/21: (/NULL - ]; /22: (/NULL - ])] + │ │ │ └── l_shipdate < l_commitdate [type=bool, outer=(20,21), constraints=(/20: (/NULL - ]; /21: (/NULL - ])] + │ │ └── filters (true) + │ └── projections + │ ├── CASE WHEN (o_orderpriority = '1-URGENT') OR (o_orderpriority = '2-HIGH') THEN 1 ELSE 0 END [type=int, outer=(6)] + │ └── CASE WHEN (o_orderpriority != '1-URGENT') AND (o_orderpriority != '2-HIGH') THEN 1 ELSE 0 END [type=int, outer=(6)] + └── aggregations + ├── sum [type=decimal, outer=(26)] + │ └── variable: column26 [type=int] + └── sum [type=decimal, outer=(28)] + └── variable: column28 [type=int] + +# -------------------------------------------------- +# Q13 +# Customer Distribution +# Seeks relationships between customers and the size of their orders. +# +# Determines the distribution of customers by the number of orders they have +# made, including customers who have no record of orders, past or present. It +# counts and reports how many customers have no orders, how many have 1, 2, 3, +# etc. A check is made to ensure that the orders counted do not fall into one of +# several special categories of orders. Special categories are identified in the +# order comment column by looking for a particular pattern. +# -------------------------------------------------- +opt +SELECT + c_count, count(*) AS custdist +FROM ( + SELECT + c_custkey, + count(o_orderkey) + FROM + customer LEFT OUTER JOIN orders ON + c_custkey = o_custkey + AND o_comment NOT LIKE '%special%requests%' + GROUP BY + c_custkey + ) AS c_orders (c_custkey, c_count) +GROUP BY + c_count +ORDER BY + custdist DESC, + c_count DESC; +---- +sort + ├── columns: c_count:18(int) custdist:19(int) + ├── key: (18) + ├── fd: (18)-->(19) + ├── ordering: -19,-18 + └── group-by + ├── columns: count:18(int) count_rows:19(int) + ├── grouping columns: count:18(int) + ├── key: (18) + ├── fd: (18)-->(19) + ├── group-by + │ ├── columns: c_custkey:1(int!null) count:18(int) + │ ├── grouping columns: c_custkey:1(int!null) + │ ├── key: (1) + │ ├── fd: (1)-->(18) + │ ├── left-join + │ │ ├── columns: c_custkey:1(int!null) o_orderkey:9(int) o_custkey:10(int) o_comment:17(string) + │ │ ├── key: (1,9) + │ │ ├── fd: (9)-->(10,17) + │ │ ├── scan customer@c_nk + │ │ │ ├── columns: c_custkey:1(int!null) + │ │ │ └── key: (1) + │ │ ├── select + │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_comment:17(string!null) + │ │ │ ├── key: (9) + │ │ │ ├── fd: (9)-->(10,17) + │ │ │ ├── scan orders + │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_comment:17(string!null) + │ │ │ │ ├── key: (9) + │ │ │ │ └── fd: (9)-->(10,17) + │ │ │ └── filters + │ │ │ └── o_comment NOT LIKE '%special%requests%' [type=bool, outer=(17), constraints=(/17: (/NULL - ])] + │ │ └── filters + │ │ └── c_custkey = o_custkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ └── aggregations + │ └── count [type=int, outer=(9)] + │ └── variable: o_orderkey [type=int] + └── aggregations + └── count-rows [type=int] + +# -------------------------------------------------- +# Q14 +# Promotion Effect +# Monitors the market response to a promotion such as TV advertisements or a +# special campaign. +# +# Determines what percentage of the revenue in a given year and month was +# derived from promotional parts. The query considers only parts actually +# shipped in that month and gives the percentage. Revenue is defined as +# (l_extendedprice * (1-l_discount)). +# -------------------------------------------------- +opt +SELECT + 100.00 * sum(CASE + WHEN p_type LIKE 'PROMO%' + THEN l_extendedprice * (1 - l_discount) + ELSE 0 + END) / sum(l_extendedprice * (1 - l_discount)) AS promo_revenue +FROM + lineitem, + part +WHERE + l_partkey = p_partkey + AND l_shipdate >= DATE '1995-09-01' + AND l_shipdate < DATE '1995-09-01' + INTERVAL '1' MONTH; +---- +project + ├── columns: promo_revenue:30(float) + ├── cardinality: [1 - 1] + ├── side-effects + ├── key: () + ├── fd: ()-->(30) + ├── scalar-group-by + │ ├── columns: sum:27(float) sum:29(float) + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(27,29) + │ ├── project + │ │ ├── columns: column26:26(float) column28:28(float) + │ │ ├── inner-join (lookup part) + │ │ │ ├── columns: l_partkey:2(int!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipdate:11(date!null) p_partkey:17(int!null) p_type:21(string!null) + │ │ │ ├── key columns: [2] = [17] + │ │ │ ├── fd: (17)-->(21), (2)==(17), (17)==(2) + │ │ │ ├── index-join lineitem + │ │ │ │ ├── columns: l_partkey:2(int!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipdate:11(date!null) + │ │ │ │ └── scan lineitem@l_sd + │ │ │ │ ├── columns: l_orderkey:1(int!null) l_linenumber:4(int!null) l_shipdate:11(date!null) + │ │ │ │ ├── constraint: /11/1/4: [/'1995-09-01' - /'1995-09-30'] + │ │ │ │ ├── key: (1,4) + │ │ │ │ └── fd: (1,4)-->(11) + │ │ │ └── filters (true) + │ │ └── projections + │ │ ├── CASE WHEN p_type LIKE 'PROMO%' THEN l_extendedprice * (1.0 - l_discount) ELSE 0.0 END [type=float, outer=(6,7,21)] + │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(6,7)] + │ └── aggregations + │ ├── sum [type=float, outer=(26)] + │ │ └── variable: column26 [type=float] + │ └── sum [type=float, outer=(28)] + │ └── variable: column28 [type=float] + └── projections + └── (sum * 100.0) / sum [type=float, outer=(27,29), side-effects] + +# -------------------------------------------------- +# Q15 +# Top Supplier +# Determines the top supplier so it can be rewarded, given more business, or +# identified for special recognition. +# +# Finds the supplier who contributed the most to the overall revenue for parts +# shipped during a given quarter of a given year. In case of a tie, the query +# lists all suppliers whose contribution was equal to the maximum, presented in +# supplier number order. +# -------------------------------------------------- +exec-ddl +CREATE VIEW revenue0 (supplier_no, total_revenue) AS + SELECT + l_suppkey, + sum(l_extendedprice * (1 - l_discount)) + FROM + lineitem + WHERE + l_shipdate >= DATE '1996-01-01' + AND l_shipdate < DATE '1996-01-01' + INTERVAL '3' MONTH + GROUP BY + l_suppkey; +---- +VIEW revenue0 (supplier_no, total_revenue) + └── SELECT l_suppkey, sum(l_extendedprice * (1 - l_discount)) FROM lineitem WHERE (l_shipdate >= DATE '1996-01-01') AND (l_shipdate < (DATE '1996-01-01' + '3 mons':::INTERVAL)) GROUP BY l_suppkey + +opt +SELECT + s_suppkey, + s_name, + s_address, + s_phone, + total_revenue +FROM + supplier, + revenue0 +WHERE + s_suppkey = supplier_no + AND total_revenue = ( + SELECT + max(total_revenue) + FROM + revenue0 + ) +ORDER BY + s_suppkey; +---- +sort + ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_phone:5(string!null) total_revenue:25(float!null) + ├── key: (1) + ├── fd: (1)-->(2,3,5,25) + ├── ordering: +1 + └── project + ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_phone:5(string!null) sum:25(float!null) + ├── key: (1) + ├── fd: (1)-->(2,3,5,25) + └── inner-join (lookup supplier) + ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_phone:5(string!null) l_suppkey:10(int!null) sum:25(float!null) + ├── key columns: [10] = [1] + ├── key: (10) + ├── fd: (1)-->(2,3,5), (10)-->(25), (1)==(10), (10)==(1) + ├── select + │ ├── columns: l_suppkey:10(int!null) sum:25(float!null) + │ ├── key: (10) + │ ├── fd: (10)-->(25) + │ ├── group-by + │ │ ├── columns: l_suppkey:10(int!null) sum:25(float) + │ │ ├── grouping columns: l_suppkey:10(int!null) + │ │ ├── key: (10) + │ │ ├── fd: (10)-->(25) + │ │ ├── project + │ │ │ ├── columns: column24:24(float) l_suppkey:10(int!null) + │ │ │ ├── index-join lineitem + │ │ │ │ ├── columns: l_suppkey:10(int!null) l_extendedprice:13(float!null) l_discount:14(float!null) l_shipdate:18(date!null) + │ │ │ │ └── scan lineitem@l_sd + │ │ │ │ ├── columns: l_orderkey:8(int!null) l_linenumber:11(int!null) l_shipdate:18(date!null) + │ │ │ │ ├── constraint: /18/8/11: [/'1996-01-01' - /'1996-03-31'] + │ │ │ │ ├── key: (8,11) + │ │ │ │ └── fd: (8,11)-->(18) + │ │ │ └── projections + │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(13,14)] + │ │ └── aggregations + │ │ └── sum [type=float, outer=(24)] + │ │ └── variable: column24 [type=float] + │ └── filters + │ └── eq [type=bool, outer=(25), constraints=(/25: (/NULL - ])] + │ ├── variable: sum [type=float] + │ └── subquery [type=float] + │ └── scalar-group-by + │ ├── columns: max:44(float) + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(44) + │ ├── group-by + │ │ ├── columns: l_suppkey:28(int!null) sum:43(float) + │ │ ├── grouping columns: l_suppkey:28(int!null) + │ │ ├── key: (28) + │ │ ├── fd: (28)-->(43) + │ │ ├── project + │ │ │ ├── columns: column42:42(float) l_suppkey:28(int!null) + │ │ │ ├── index-join lineitem + │ │ │ │ ├── columns: l_suppkey:28(int!null) l_extendedprice:31(float!null) l_discount:32(float!null) l_shipdate:36(date!null) + │ │ │ │ └── scan lineitem@l_sd + │ │ │ │ ├── columns: l_orderkey:26(int!null) l_linenumber:29(int!null) l_shipdate:36(date!null) + │ │ │ │ ├── constraint: /36/26/29: [/'1996-01-01' - /'1996-03-31'] + │ │ │ │ ├── key: (26,29) + │ │ │ │ └── fd: (26,29)-->(36) + │ │ │ └── projections + │ │ │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(31,32)] + │ │ └── aggregations + │ │ └── sum [type=float, outer=(42)] + │ │ └── variable: column42 [type=float] + │ └── aggregations + │ └── max [type=float, outer=(43)] + │ └── variable: sum [type=float] + └── filters (true) + +# -------------------------------------------------- +# Q16 +# Parts/Supplier Relationship +# Finds out how many suppliers can supply parts with given attributes. It might +# be used, for example, to determine whether there is a sufficient number of +# suppliers for heavily ordered parts. +# +# Counts the number of suppliers who can supply parts that satisfy a particular +# customer's requirements. The customer is interested in parts of eight +# different sizes as long as they are not of a given type, not of a given brand, +# and not from a supplier who has had complaints registered at the Better +# Business Bureau. Results must be presented in descending count and ascending +# brand, type, and size. +# -------------------------------------------------- +opt +SELECT + p_brand, + p_type, + p_size, + count(DISTINCT ps_suppkey) AS supplier_cnt +FROM + partsupp, + part +WHERE + p_partkey = ps_partkey + AND p_brand <> 'Brand#45' + AND p_type NOT LIKE 'MEDIUM POLISHED %' + AND p_size IN (49, 14, 23, 45, 19, 3, 36, 9) + AND ps_suppkey NOT IN ( + SELECT + s_suppkey + FROM + supplier + WHERE + s_comment LIKE '%Customer%Complaints%' + ) +GROUP BY + p_brand, + p_type, + p_size +ORDER BY + supplier_cnt DESC, + p_brand, + p_type, + p_size; +---- +sort + ├── columns: p_brand:9(string!null) p_type:10(string!null) p_size:11(int!null) supplier_cnt:22(int) + ├── key: (9-11) + ├── fd: (9-11)-->(22) + ├── ordering: -22,+9,+10,+11 + └── group-by + ├── columns: p_brand:9(string!null) p_type:10(string!null) p_size:11(int!null) count:22(int) + ├── grouping columns: p_brand:9(string!null) p_type:10(string!null) p_size:11(int!null) + ├── key: (9-11) + ├── fd: (9-11)-->(22) + ├── inner-join + │ ├── columns: ps_partkey:1(int!null) ps_suppkey:2(int!null) p_partkey:6(int!null) p_brand:9(string!null) p_type:10(string!null) p_size:11(int!null) + │ ├── key: (2,6) + │ ├── fd: (6)-->(9-11), (1)==(6), (6)==(1) + │ ├── anti-join (merge) + │ │ ├── columns: ps_partkey:1(int!null) ps_suppkey:2(int!null) + │ │ ├── left ordering: +2 + │ │ ├── right ordering: +15 + │ │ ├── key: (1,2) + │ │ ├── scan partsupp@ps_sk + │ │ │ ├── columns: ps_partkey:1(int!null) ps_suppkey:2(int!null) + │ │ │ ├── key: (1,2) + │ │ │ └── ordering: +2 + │ │ ├── select + │ │ │ ├── columns: s_suppkey:15(int!null) s_comment:21(string!null) + │ │ │ ├── key: (15) + │ │ │ ├── fd: (15)-->(21) + │ │ │ ├── ordering: +15 + │ │ │ ├── scan supplier + │ │ │ │ ├── columns: s_suppkey:15(int!null) s_comment:21(string!null) + │ │ │ │ ├── key: (15) + │ │ │ │ ├── fd: (15)-->(21) + │ │ │ │ └── ordering: +15 + │ │ │ └── filters + │ │ │ └── s_comment LIKE '%Customer%Complaints%' [type=bool, outer=(21), constraints=(/21: (/NULL - ])] + │ │ └── filters (true) + │ ├── select + │ │ ├── columns: p_partkey:6(int!null) p_brand:9(string!null) p_type:10(string!null) p_size:11(int!null) + │ │ ├── key: (6) + │ │ ├── fd: (6)-->(9-11) + │ │ ├── scan part + │ │ │ ├── columns: p_partkey:6(int!null) p_brand:9(string!null) p_type:10(string!null) p_size:11(int!null) + │ │ │ ├── key: (6) + │ │ │ └── fd: (6)-->(9-11) + │ │ └── filters + │ │ ├── p_brand != 'Brand#45' [type=bool, outer=(9), constraints=(/9: (/NULL - /'Brand#45') [/e'Brand#45\x00' - ]; tight)] + │ │ ├── p_type NOT LIKE 'MEDIUM POLISHED %' [type=bool, outer=(10), constraints=(/10: (/NULL - ])] + │ │ └── p_size IN (3, 9, 14, 19, 23, 36, 45, 49) [type=bool, outer=(11), constraints=(/11: [/3 - /3] [/9 - /9] [/14 - /14] [/19 - /19] [/23 - /23] [/36 - /36] [/45 - /45] [/49 - /49]; tight)] + │ └── filters + │ └── p_partkey = ps_partkey [type=bool, outer=(1,6), constraints=(/1: (/NULL - ]; /6: (/NULL - ]), fd=(1)==(6), (6)==(1)] + └── aggregations + └── count [type=int, outer=(2)] + └── agg-distinct [type=int] + └── variable: ps_suppkey [type=int] + +# -------------------------------------------------- +# Q17 +# Small-Quantity-Order Revenue +# Determines how much average yearly revenue would be lost if orders were no +# longer filled for small quantities of certain parts. This may reduce overhead +# expenses by concentrating sales on larger shipments. +# +# Considers parts of a given brand and with a given container type and +# determines the average lineitem quantity of such parts ordered for all orders +# (past and pending) in the 7-year database. What would be the average yearly +# gross (undiscounted) loss in revenue if orders for these parts with a quantity +# of less than 20% of this average were no longer taken? +# +# TODO: +# 1. Allow Select to be pushed below RowNumber used to add key column +# -------------------------------------------------- +opt +SELECT + sum(l_extendedprice) / 7.0 AS avg_yearly +FROM + lineitem, + part +WHERE + p_partkey = l_partkey + AND p_brand = 'Brand#23' + AND p_container = 'MED BOX' + AND l_quantity < ( + SELECT + 0.2 * avg(l_quantity) + FROM + lineitem + WHERE + l_partkey = p_partkey + ); +---- +project + ├── columns: avg_yearly:45(float) + ├── cardinality: [1 - 1] + ├── side-effects + ├── key: () + ├── fd: ()-->(45) + ├── scalar-group-by + │ ├── columns: sum:44(float) + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(44) + │ ├── inner-join (lookup lineitem) + │ │ ├── columns: l_partkey:2(int!null) l_quantity:5(float!null) l_extendedprice:6(float!null) p_partkey:17(int!null) "?column?":43(float!null) + │ │ ├── key columns: [1 4] = [1 4] + │ │ ├── fd: (17)-->(43), (2)==(17), (17)==(2) + │ │ ├── inner-join (lookup lineitem@l_pk) + │ │ │ ├── columns: l_orderkey:1(int!null) l_partkey:2(int!null) l_linenumber:4(int!null) p_partkey:17(int!null) "?column?":43(float) + │ │ │ ├── key columns: [17] = [2] + │ │ │ ├── key: (1,4) + │ │ │ ├── fd: (17)-->(43), (1,4)-->(2), (2)==(17), (17)==(2) + │ │ │ ├── project + │ │ │ │ ├── columns: "?column?":43(float) p_partkey:17(int!null) + │ │ │ │ ├── key: (17) + │ │ │ │ ├── fd: (17)-->(43) + │ │ │ │ ├── group-by + │ │ │ │ │ ├── columns: p_partkey:17(int!null) avg:42(float) + │ │ │ │ │ ├── grouping columns: p_partkey:17(int!null) + │ │ │ │ │ ├── internal-ordering: +17 opt(20,23) + │ │ │ │ │ ├── key: (17) + │ │ │ │ │ ├── fd: (17)-->(42) + │ │ │ │ │ ├── left-join (lookup lineitem) + │ │ │ │ │ │ ├── columns: p_partkey:17(int!null) p_brand:20(string!null) p_container:23(string!null) l_partkey:27(int) l_quantity:30(float) + │ │ │ │ │ │ ├── key columns: [26 29] = [26 29] + │ │ │ │ │ │ ├── fd: ()-->(20,23) + │ │ │ │ │ │ ├── ordering: +17 opt(20,23) [actual: +17] + │ │ │ │ │ │ ├── left-join (lookup lineitem@l_pk) + │ │ │ │ │ │ │ ├── columns: p_partkey:17(int!null) p_brand:20(string!null) p_container:23(string!null) l_orderkey:26(int) l_partkey:27(int) l_linenumber:29(int) + │ │ │ │ │ │ │ ├── key columns: [17] = [27] + │ │ │ │ │ │ │ ├── key: (17,26,29) + │ │ │ │ │ │ │ ├── fd: ()-->(20,23), (26,29)-->(27) + │ │ │ │ │ │ │ ├── ordering: +17 opt(20,23) [actual: +17] + │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ │ ├── columns: p_partkey:17(int!null) p_brand:20(string!null) p_container:23(string!null) + │ │ │ │ │ │ │ │ ├── key: (17) + │ │ │ │ │ │ │ │ ├── fd: ()-->(20,23) + │ │ │ │ │ │ │ │ ├── ordering: +17 opt(20,23) [actual: +17] + │ │ │ │ │ │ │ │ ├── scan part + │ │ │ │ │ │ │ │ │ ├── columns: p_partkey:17(int!null) p_brand:20(string!null) p_container:23(string!null) + │ │ │ │ │ │ │ │ │ ├── key: (17) + │ │ │ │ │ │ │ │ │ ├── fd: (17)-->(20,23) + │ │ │ │ │ │ │ │ │ └── ordering: +17 opt(20,23) [actual: +17] + │ │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ │ ├── p_brand = 'Brand#23' [type=bool, outer=(20), constraints=(/20: [/'Brand#23' - /'Brand#23']; tight), fd=()-->(20)] + │ │ │ │ │ │ │ │ └── p_container = 'MED BOX' [type=bool, outer=(23), constraints=(/23: [/'MED BOX' - /'MED BOX']; tight), fd=()-->(23)] + │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ └── aggregations + │ │ │ │ │ └── avg [type=float, outer=(30)] + │ │ │ │ │ └── variable: l_quantity [type=float] + │ │ │ │ └── projections + │ │ │ │ └── avg * 0.2 [type=float, outer=(42)] + │ │ │ └── filters (true) + │ │ └── filters + │ │ └── l_quantity < ?column? [type=bool, outer=(5,43), constraints=(/5: (/NULL - ]; /43: (/NULL - ])] + │ └── aggregations + │ └── sum [type=float, outer=(6)] + │ └── variable: l_extendedprice [type=float] + └── projections + └── sum / 7.0 [type=float, outer=(44), side-effects] + +# -------------------------------------------------- +# Q18 +# Large Volume Customer +# Ranks customers based on their having placed a large quantity order. Large +# quantity orders are defined as those orders whose total quantity is above a +# certain level. +# +# Finds a list of the top 100 customers who have ever placed large quantity +# orders. The query lists the customer name, customer key, the order key, date +# and total price and the quantity for the order. +# -------------------------------------------------- +opt +SELECT + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +FROM + customer, + orders, + lineitem +WHERE + o_orderkey IN ( + SELECT + l_orderkey + FROM + lineitem + GROUP BY + l_orderkey HAVING + sum(l_quantity) > 300 + ) + AND c_custkey = o_custkey + AND o_orderkey = l_orderkey +GROUP BY + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +ORDER BY + o_totalprice DESC, + o_orderdate +LIMIT 100; +---- +limit + ├── columns: c_name:2(string) c_custkey:1(int) o_orderkey:9(int!null) o_orderdate:13(date) o_totalprice:12(float) sum:51(float) + ├── internal-ordering: -12,+13 + ├── cardinality: [0 - 100] + ├── key: (9) + ├── fd: (1)-->(2), (9)-->(1,2,12,13,51) + ├── ordering: -12,+13 + ├── sort + │ ├── columns: c_custkey:1(int) c_name:2(string) o_orderkey:9(int!null) o_totalprice:12(float) o_orderdate:13(date) sum:51(float) + │ ├── key: (9) + │ ├── fd: (1)-->(2), (9)-->(1,2,12,13,51) + │ ├── ordering: -12,+13 + │ └── group-by + │ ├── columns: c_custkey:1(int) c_name:2(string) o_orderkey:9(int!null) o_totalprice:12(float) o_orderdate:13(date) sum:51(float) + │ ├── grouping columns: o_orderkey:9(int!null) + │ ├── key: (9) + │ ├── fd: (1)-->(2), (9)-->(1,2,12,13,51) + │ ├── inner-join + │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) o_orderkey:9(int!null) o_custkey:10(int!null) o_totalprice:12(float!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_quantity:22(float!null) + │ │ ├── fd: (1)-->(2), (9)-->(10,12,13), (9)==(18), (18)==(9), (1)==(10), (10)==(1) + │ │ ├── scan customer + │ │ │ ├── columns: c_custkey:1(int!null) c_name:2(string!null) + │ │ │ ├── key: (1) + │ │ │ └── fd: (1)-->(2) + │ │ ├── inner-join (merge) + │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_totalprice:12(float!null) o_orderdate:13(date!null) l_orderkey:18(int!null) l_quantity:22(float!null) + │ │ │ ├── left ordering: +9 + │ │ │ ├── right ordering: +18 + │ │ │ ├── fd: (9)-->(10,12,13), (9)==(18), (18)==(9) + │ │ │ ├── semi-join (merge) + │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_totalprice:12(float!null) o_orderdate:13(date!null) + │ │ │ │ ├── left ordering: +9 + │ │ │ │ ├── right ordering: +34 + │ │ │ │ ├── key: (9) + │ │ │ │ ├── fd: (9)-->(10,12,13) + │ │ │ │ ├── ordering: +9 + │ │ │ │ ├── scan orders + │ │ │ │ │ ├── columns: o_orderkey:9(int!null) o_custkey:10(int!null) o_totalprice:12(float!null) o_orderdate:13(date!null) + │ │ │ │ │ ├── key: (9) + │ │ │ │ │ ├── fd: (9)-->(10,12,13) + │ │ │ │ │ └── ordering: +9 + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: l_orderkey:34(int!null) sum:50(float!null) + │ │ │ │ │ ├── key: (34) + │ │ │ │ │ ├── fd: (34)-->(50) + │ │ │ │ │ ├── ordering: +34 + │ │ │ │ │ ├── group-by + │ │ │ │ │ │ ├── columns: l_orderkey:34(int!null) sum:50(float) + │ │ │ │ │ │ ├── grouping columns: l_orderkey:34(int!null) + │ │ │ │ │ │ ├── key: (34) + │ │ │ │ │ │ ├── fd: (34)-->(50) + │ │ │ │ │ │ ├── ordering: +34 + │ │ │ │ │ │ ├── scan lineitem + │ │ │ │ │ │ │ ├── columns: l_orderkey:34(int!null) l_quantity:38(float!null) + │ │ │ │ │ │ │ └── ordering: +34 + │ │ │ │ │ │ └── aggregations + │ │ │ │ │ │ └── sum [type=float, outer=(38)] + │ │ │ │ │ │ └── variable: l_quantity [type=float] + │ │ │ │ │ └── filters + │ │ │ │ │ └── sum > 300.0 [type=bool, outer=(50), constraints=(/50: [/300.00000000000006 - ]; tight)] + │ │ │ │ └── filters (true) + │ │ │ ├── scan lineitem + │ │ │ │ ├── columns: l_orderkey:18(int!null) l_quantity:22(float!null) + │ │ │ │ └── ordering: +18 + │ │ │ └── filters (true) + │ │ └── filters + │ │ └── c_custkey = o_custkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ └── aggregations + │ ├── sum [type=float, outer=(22)] + │ │ └── variable: l_quantity [type=float] + │ ├── const-agg [type=int, outer=(1)] + │ │ └── variable: c_custkey [type=int] + │ ├── const-agg [type=string, outer=(2)] + │ │ └── variable: c_name [type=string] + │ ├── const-agg [type=float, outer=(12)] + │ │ └── variable: o_totalprice [type=float] + │ └── const-agg [type=date, outer=(13)] + │ └── variable: o_orderdate [type=date] + └── const: 100 [type=int] + +# -------------------------------------------------- +# Q19 +# Discounted Revenue +# Reports the gross discounted revenue attributed to the sale of selected parts +# handled in a particular manner. This query is an example of code such as might +# be produced programmatically by a data mining tool. +# +# The Discounted Revenue query finds the gross discounted revenue for all orders +# for three different types of parts that were shipped by air and delivered in +# person. Parts are selected based on the combination of specific brands, a list +# of containers, and a range of sizes. +# -------------------------------------------------- +opt +SELECT + sum(l_extendedprice* (1 - l_discount)) AS revenue +FROM + lineitem, + part +WHERE + ( + p_partkey = l_partkey + AND p_brand = 'Brand#12' + AND p_container IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') + AND l_quantity >= 1 AND l_quantity <= 1 + 10 + AND p_size BETWEEN 1 AND 5 + AND l_shipmode IN ('AIR', 'AIR REG') + AND l_shipinstruct = 'DELIVER IN PERSON' + ) + OR + ( + p_partkey = l_partkey + AND p_brand = 'Brand#23' + AND p_container IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') + AND l_quantity >= 10 AND l_quantity <= 10 + 10 + AND p_size BETWEEN 1 AND 10 + AND l_shipmode IN ('AIR', 'AIR REG') + AND l_shipinstruct = 'DELIVER IN PERSON' + ) + OR + ( + p_partkey = l_partkey + AND p_brand = 'Brand#34' + AND p_container IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') + AND l_quantity >= 20 AND l_quantity <= 20 + 10 + AND p_size BETWEEN 1 AND 15 + AND l_shipmode IN ('AIR', 'AIR REG') + AND l_shipinstruct = 'DELIVER IN PERSON' + ); +---- +scalar-group-by + ├── columns: revenue:27(float) + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(27) + ├── project + │ ├── columns: column26:26(float) + │ ├── inner-join (lookup part) + │ │ ├── columns: l_partkey:2(int!null) l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipinstruct:14(string!null) l_shipmode:15(string!null) p_partkey:17(int!null) p_brand:20(string!null) p_size:22(int!null) p_container:23(string!null) + │ │ ├── key columns: [2] = [17] + │ │ ├── fd: ()-->(14), (17)-->(20,22,23), (2)==(17), (17)==(2) + │ │ ├── select + │ │ │ ├── columns: l_partkey:2(int!null) l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipinstruct:14(string!null) l_shipmode:15(string!null) + │ │ │ ├── fd: ()-->(14) + │ │ │ ├── scan lineitem + │ │ │ │ └── columns: l_partkey:2(int!null) l_quantity:5(float!null) l_extendedprice:6(float!null) l_discount:7(float!null) l_shipinstruct:14(string!null) l_shipmode:15(string!null) + │ │ │ └── filters + │ │ │ ├── l_shipmode IN ('AIR', 'AIR REG') [type=bool, outer=(15), constraints=(/15: [/'AIR' - /'AIR'] [/'AIR REG' - /'AIR REG']; tight)] + │ │ │ └── l_shipinstruct = 'DELIVER IN PERSON' [type=bool, outer=(14), constraints=(/14: [/'DELIVER IN PERSON' - /'DELIVER IN PERSON']; tight), fd=()-->(14)] + │ │ └── filters + │ │ ├── ((((((p_brand = 'Brand#12') AND (p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'))) AND (l_quantity >= 1.0)) AND (l_quantity <= 11.0)) AND (p_size <= 5)) OR (((((p_brand = 'Brand#23') AND (p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'))) AND (l_quantity >= 10.0)) AND (l_quantity <= 20.0)) AND (p_size <= 10))) OR (((((p_brand = 'Brand#34') AND (p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'))) AND (l_quantity >= 20.0)) AND (l_quantity <= 30.0)) AND (p_size <= 15)) [type=bool, outer=(5,20,22,23)] + │ │ └── p_size >= 1 [type=bool, outer=(22), constraints=(/22: [/1 - ]; tight)] + │ └── projections + │ └── l_extendedprice * (1.0 - l_discount) [type=float, outer=(6,7)] + └── aggregations + └── sum [type=float, outer=(26)] + └── variable: column26 [type=float] + +# -------------------------------------------------- +# Q20 +# Potential Part Promotion +# Identifies suppliers in a particular nation having selected parts that may be +# candidates for a promotional offer. +# +# Identifies suppliers who have an excess of a given part available; an excess +# defined to be more than 50% of the parts like the given part that the supplier +# shipped in a given year for a given nation. Only parts whose names share a +# certain naming convention are considered. +# +# TODO: +# 1. Push 'forest%' prefix filter down into Scan +# -------------------------------------------------- +opt +SELECT + s_name, + s_address +FROM + supplier, + nation +WHERE + s_suppkey IN ( + SELECT + ps_suppkey + FROM + partsupp + WHERE + ps_partkey IN ( + SELECT + p_partkey + FROM + part + WHERE + p_name LIKE 'forest%' + ) + AND ps_availqty > ( + SELECT + 0.5 * sum(l_quantity) + FROM + lineitem + WHERE + l_partkey = ps_partkey + AND l_suppkey = ps_suppkey + AND l_shipdate >= DATE '1994-01-01' + AND l_shipdate < DATE '1994-01-01' + INTERVAL '1' YEAR + ) + ) + AND s_nationkey = n_nationkey + AND n_name = 'CANADA' +ORDER BY + s_name; +---- +sort + ├── columns: s_name:2(string!null) s_address:3(string!null) + ├── ordering: +2 + └── project + ├── columns: s_name:2(string!null) s_address:3(string!null) + └── inner-join + ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_nationkey:4(int!null) n_nationkey:8(int!null) n_name:9(string!null) + ├── key: (1) + ├── fd: ()-->(9), (1)-->(2-4), (4)==(8), (8)==(4) + ├── semi-join + │ ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_nationkey:4(int!null) + │ ├── key: (1) + │ ├── fd: (1)-->(2-4) + │ ├── scan supplier + │ │ ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_address:3(string!null) s_nationkey:4(int!null) + │ │ ├── key: (1) + │ │ └── fd: (1)-->(2-4) + │ ├── semi-join + │ │ ├── columns: ps_partkey:12(int!null) ps_suppkey:13(int!null) + │ │ ├── key: (12,13) + │ │ ├── project + │ │ │ ├── columns: ps_partkey:12(int!null) ps_suppkey:13(int!null) + │ │ │ ├── key: (12,13) + │ │ │ └── select + │ │ │ ├── columns: ps_partkey:12(int!null) ps_suppkey:13(int!null) ps_availqty:14(int!null) sum:42(float) + │ │ │ ├── key: (12,13) + │ │ │ ├── fd: (12,13)-->(14,42) + │ │ │ ├── group-by + │ │ │ │ ├── columns: ps_partkey:12(int!null) ps_suppkey:13(int!null) ps_availqty:14(int) sum:42(float) + │ │ │ │ ├── grouping columns: ps_partkey:12(int!null) ps_suppkey:13(int!null) + │ │ │ │ ├── key: (12,13) + │ │ │ │ ├── fd: (12,13)-->(14,42) + │ │ │ │ ├── left-join + │ │ │ │ │ ├── columns: ps_partkey:12(int!null) ps_suppkey:13(int!null) ps_availqty:14(int!null) l_partkey:27(int) l_suppkey:28(int) l_quantity:30(float) l_shipdate:36(date) + │ │ │ │ │ ├── fd: (12,13)-->(14) + │ │ │ │ │ ├── scan partsupp + │ │ │ │ │ │ ├── columns: ps_partkey:12(int!null) ps_suppkey:13(int!null) ps_availqty:14(int!null) + │ │ │ │ │ │ ├── key: (12,13) + │ │ │ │ │ │ └── fd: (12,13)-->(14) + │ │ │ │ │ ├── index-join lineitem + │ │ │ │ │ │ ├── columns: l_partkey:27(int!null) l_suppkey:28(int!null) l_quantity:30(float!null) l_shipdate:36(date!null) + │ │ │ │ │ │ └── scan lineitem@l_sd + │ │ │ │ │ │ ├── columns: l_orderkey:26(int!null) l_linenumber:29(int!null) l_shipdate:36(date!null) + │ │ │ │ │ │ ├── constraint: /36/26/29: [/'1994-01-01' - /'1994-12-31'] + │ │ │ │ │ │ ├── key: (26,29) + │ │ │ │ │ │ └── fd: (26,29)-->(36) + │ │ │ │ │ └── filters + │ │ │ │ │ ├── l_partkey = ps_partkey [type=bool, outer=(12,27), constraints=(/12: (/NULL - ]; /27: (/NULL - ]), fd=(12)==(27), (27)==(12)] + │ │ │ │ │ └── l_suppkey = ps_suppkey [type=bool, outer=(13,28), constraints=(/13: (/NULL - ]; /28: (/NULL - ]), fd=(13)==(28), (28)==(13)] + │ │ │ │ └── aggregations + │ │ │ │ ├── sum [type=float, outer=(30)] + │ │ │ │ │ └── variable: l_quantity [type=float] + │ │ │ │ └── const-agg [type=int, outer=(14)] + │ │ │ │ └── variable: ps_availqty [type=int] + │ │ │ └── filters + │ │ │ └── ps_availqty > (sum * 0.5) [type=bool, outer=(14,42), constraints=(/14: (/NULL - ])] + │ │ ├── select + │ │ │ ├── columns: p_partkey:17(int!null) p_name:18(string!null) + │ │ │ ├── key: (17) + │ │ │ ├── fd: (17)-->(18) + │ │ │ ├── scan part + │ │ │ │ ├── columns: p_partkey:17(int!null) p_name:18(string!null) + │ │ │ │ ├── key: (17) + │ │ │ │ └── fd: (17)-->(18) + │ │ │ └── filters + │ │ │ └── p_name LIKE 'forest%' [type=bool, outer=(18), constraints=(/18: [/'forest' - /'foresu'); tight)] + │ │ └── filters + │ │ └── ps_partkey = p_partkey [type=bool, outer=(12,17), constraints=(/12: (/NULL - ]; /17: (/NULL - ]), fd=(12)==(17), (17)==(12)] + │ └── filters + │ └── s_suppkey = ps_suppkey [type=bool, outer=(1,13), constraints=(/1: (/NULL - ]; /13: (/NULL - ]), fd=(1)==(13), (13)==(1)] + ├── select + │ ├── columns: n_nationkey:8(int!null) n_name:9(string!null) + │ ├── key: (8) + │ ├── fd: ()-->(9) + │ ├── scan nation + │ │ ├── columns: n_nationkey:8(int!null) n_name:9(string!null) + │ │ ├── key: (8) + │ │ └── fd: (8)-->(9) + │ └── filters + │ └── n_name = 'CANADA' [type=bool, outer=(9), constraints=(/9: [/'CANADA' - /'CANADA']; tight), fd=()-->(9)] + └── filters + └── s_nationkey = n_nationkey [type=bool, outer=(4,8), constraints=(/4: (/NULL - ]; /8: (/NULL - ]), fd=(4)==(8), (8)==(4)] + +# -------------------------------------------------- +# Q21 +# Suppliers Who Kept Orders Waiting Query +# Identifies certain suppliers who were not able to ship required parts in a +# timely manner. +# +# Identifies suppliers, for a given nation, whose product was part of a multi- +# supplier order (with current status of 'F') where they were the only supplier +# who failed to meet the committed delivery date. +# -------------------------------------------------- +opt +SELECT + s_name, + count(*) AS numwait +FROM + supplier, + lineitem l1, + orders, + nation +WHERE + s_suppkey = l1.l_suppkey + AND o_orderkey = l1.l_orderkey + AND o_orderstatus = 'F' + AND l1.l_receiptDATE > l1.l_commitdate + AND EXISTS ( + SELECT + * + FROM + lineitem l2 + WHERE + l2.l_orderkey = l1.l_orderkey + AND l2.l_suppkey <> l1.l_suppkey + ) + AND NOT EXISTS ( + SELECT + * + FROM + lineitem l3 + WHERE + l3.l_orderkey = l1.l_orderkey + AND l3.l_suppkey <> l1.l_suppkey + AND l3.l_receiptDATE > l3.l_commitdate + ) + AND s_nationkey = n_nationkey + AND n_name = 'SAUDI ARABIA' +GROUP BY + s_name +ORDER BY + numwait DESC, + s_name +LIMIT 100; +---- +limit + ├── columns: s_name:2(string!null) numwait:69(int) + ├── internal-ordering: -69,+2 + ├── cardinality: [0 - 100] + ├── key: (2) + ├── fd: (2)-->(69) + ├── ordering: -69,+2 + ├── sort + │ ├── columns: s_name:2(string!null) count_rows:69(int) + │ ├── key: (2) + │ ├── fd: (2)-->(69) + │ ├── ordering: -69,+2 + │ └── group-by + │ ├── columns: s_name:2(string!null) count_rows:69(int) + │ ├── grouping columns: s_name:2(string!null) + │ ├── key: (2) + │ ├── fd: (2)-->(69) + │ ├── inner-join + │ │ ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_nationkey:4(int!null) l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) o_orderkey:24(int!null) o_orderstatus:26(string!null) n_nationkey:33(int!null) n_name:34(string!null) + │ │ ├── fd: ()-->(26,34), (1)-->(2,4), (8)==(24), (24)==(8), (1)==(10), (10)==(1), (4)==(33), (33)==(4) + │ │ ├── scan supplier + │ │ │ ├── columns: s_suppkey:1(int!null) s_name:2(string!null) s_nationkey:4(int!null) + │ │ │ ├── key: (1) + │ │ │ └── fd: (1)-->(2,4) + │ │ ├── inner-join + │ │ │ ├── columns: l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) o_orderkey:24(int!null) o_orderstatus:26(string!null) n_nationkey:33(int!null) n_name:34(string!null) + │ │ │ ├── fd: ()-->(26,34), (8)==(24), (24)==(8) + │ │ │ ├── semi-join (merge) + │ │ │ │ ├── columns: l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) + │ │ │ │ ├── left ordering: +8 + │ │ │ │ ├── right ordering: +37 + │ │ │ │ ├── anti-join (merge) + │ │ │ │ │ ├── columns: l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) + │ │ │ │ │ ├── left ordering: +8 + │ │ │ │ │ ├── right ordering: +53 + │ │ │ │ │ ├── ordering: +8 + │ │ │ │ │ ├── select + │ │ │ │ │ │ ├── columns: l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) + │ │ │ │ │ │ ├── ordering: +8 + │ │ │ │ │ │ ├── scan l1 + │ │ │ │ │ │ │ ├── columns: l1.l_orderkey:8(int!null) l1.l_suppkey:10(int!null) l1.l_commitdate:19(date!null) l1.l_receiptdate:20(date!null) + │ │ │ │ │ │ │ └── ordering: +8 + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── l1.l_receiptdate > l1.l_commitdate [type=bool, outer=(19,20), constraints=(/19: (/NULL - ]; /20: (/NULL - ])] + │ │ │ │ │ ├── select + │ │ │ │ │ │ ├── columns: l3.l_orderkey:53(int!null) l3.l_partkey:54(int!null) l3.l_suppkey:55(int!null) l3.l_linenumber:56(int!null) l3.l_quantity:57(float!null) l3.l_extendedprice:58(float!null) l3.l_discount:59(float!null) l3.l_tax:60(float!null) l3.l_returnflag:61(string!null) l3.l_linestatus:62(string!null) l3.l_shipdate:63(date!null) l3.l_commitdate:64(date!null) l3.l_receiptdate:65(date!null) l3.l_shipinstruct:66(string!null) l3.l_shipmode:67(string!null) l3.l_comment:68(string!null) + │ │ │ │ │ │ ├── key: (53,56) + │ │ │ │ │ │ ├── fd: (53,56)-->(54,55,57-68) + │ │ │ │ │ │ ├── ordering: +53 + │ │ │ │ │ │ ├── scan l3 + │ │ │ │ │ │ │ ├── columns: l3.l_orderkey:53(int!null) l3.l_partkey:54(int!null) l3.l_suppkey:55(int!null) l3.l_linenumber:56(int!null) l3.l_quantity:57(float!null) l3.l_extendedprice:58(float!null) l3.l_discount:59(float!null) l3.l_tax:60(float!null) l3.l_returnflag:61(string!null) l3.l_linestatus:62(string!null) l3.l_shipdate:63(date!null) l3.l_commitdate:64(date!null) l3.l_receiptdate:65(date!null) l3.l_shipinstruct:66(string!null) l3.l_shipmode:67(string!null) l3.l_comment:68(string!null) + │ │ │ │ │ │ │ ├── key: (53,56) + │ │ │ │ │ │ │ ├── fd: (53,56)-->(54,55,57-68) + │ │ │ │ │ │ │ └── ordering: +53 + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── l3.l_receiptdate > l3.l_commitdate [type=bool, outer=(64,65), constraints=(/64: (/NULL - ]; /65: (/NULL - ])] + │ │ │ │ │ └── filters + │ │ │ │ │ └── l3.l_suppkey != l1.l_suppkey [type=bool, outer=(10,55), constraints=(/10: (/NULL - ]; /55: (/NULL - ])] + │ │ │ │ ├── scan l2 + │ │ │ │ │ ├── columns: l2.l_orderkey:37(int!null) l2.l_partkey:38(int!null) l2.l_suppkey:39(int!null) l2.l_linenumber:40(int!null) l2.l_quantity:41(float!null) l2.l_extendedprice:42(float!null) l2.l_discount:43(float!null) l2.l_tax:44(float!null) l2.l_returnflag:45(string!null) l2.l_linestatus:46(string!null) l2.l_shipdate:47(date!null) l2.l_commitdate:48(date!null) l2.l_receiptdate:49(date!null) l2.l_shipinstruct:50(string!null) l2.l_shipmode:51(string!null) l2.l_comment:52(string!null) + │ │ │ │ │ ├── key: (37,40) + │ │ │ │ │ ├── fd: (37,40)-->(38,39,41-52) + │ │ │ │ │ └── ordering: +37 + │ │ │ │ └── filters + │ │ │ │ └── l2.l_suppkey != l1.l_suppkey [type=bool, outer=(10,39), constraints=(/10: (/NULL - ]; /39: (/NULL - ])] + │ │ │ ├── inner-join + │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) n_nationkey:33(int!null) n_name:34(string!null) + │ │ │ │ ├── key: (24,33) + │ │ │ │ ├── fd: ()-->(26,34) + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) + │ │ │ │ │ ├── key: (24) + │ │ │ │ │ ├── fd: ()-->(26) + │ │ │ │ │ ├── scan orders + │ │ │ │ │ │ ├── columns: o_orderkey:24(int!null) o_orderstatus:26(string!null) + │ │ │ │ │ │ ├── key: (24) + │ │ │ │ │ │ └── fd: (24)-->(26) + │ │ │ │ │ └── filters + │ │ │ │ │ └── o_orderstatus = 'F' [type=bool, outer=(26), constraints=(/26: [/'F' - /'F']; tight), fd=()-->(26)] + │ │ │ │ ├── select + │ │ │ │ │ ├── columns: n_nationkey:33(int!null) n_name:34(string!null) + │ │ │ │ │ ├── key: (33) + │ │ │ │ │ ├── fd: ()-->(34) + │ │ │ │ │ ├── scan nation + │ │ │ │ │ │ ├── columns: n_nationkey:33(int!null) n_name:34(string!null) + │ │ │ │ │ │ ├── key: (33) + │ │ │ │ │ │ └── fd: (33)-->(34) + │ │ │ │ │ └── filters + │ │ │ │ │ └── n_name = 'SAUDI ARABIA' [type=bool, outer=(34), constraints=(/34: [/'SAUDI ARABIA' - /'SAUDI ARABIA']; tight), fd=()-->(34)] + │ │ │ │ └── filters (true) + │ │ │ └── filters + │ │ │ └── o_orderkey = l1.l_orderkey [type=bool, outer=(8,24), constraints=(/8: (/NULL - ]; /24: (/NULL - ]), fd=(8)==(24), (24)==(8)] + │ │ └── filters + │ │ ├── s_suppkey = l1.l_suppkey [type=bool, outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ └── s_nationkey = n_nationkey [type=bool, outer=(4,33), constraints=(/4: (/NULL - ]; /33: (/NULL - ]), fd=(4)==(33), (33)==(4)] + │ └── aggregations + │ └── count-rows [type=int] + └── const: 100 [type=int] + +# -------------------------------------------------- +# Q22 +# Global Sales Opportunity +# Identifies geographies where there are customers who may be likely to make a +# purchase. +# +# This query counts how many customers within a specific range of country codes +# have not placed orders for 7 years but who have a greater than average +# “positive” account balance. It also reflects the magnitude of that balance. +# Country code is defined as the first two characters of c_phone. +# -------------------------------------------------- +opt +SELECT + cntrycode, + count(*) AS numcust, + sum(c_acctbal) AS totacctbal +FROM ( + SELECT + substring(c_phone FROM 1 FOR 2) AS cntrycode, + c_acctbal + FROM + customer + WHERE + substring(c_phone FROM 1 FOR 2) in + ('13', '31', '23', '29', '30', '18', '17') + AND c_acctbal > ( + SELECT + avg(c_acctbal) + FROM + customer + WHERE + c_acctbal > 0.00 + AND substring(c_phone FROM 1 FOR 2) in + ('13', '31', '23', '29', '30', '18', '17') + ) + AND NOT EXISTS ( + SELECT + * + FROM + orders + WHERE + o_custkey = c_custkey + ) + ) AS custsale +GROUP BY + cntrycode +ORDER BY + cntrycode; +---- +sort + ├── columns: cntrycode:27(string) numcust:28(int) totacctbal:29(float) + ├── key: (27) + ├── fd: (27)-->(28,29) + ├── ordering: +27 + └── group-by + ├── columns: cntrycode:27(string) count_rows:28(int) sum:29(float) + ├── grouping columns: cntrycode:27(string) + ├── key: (27) + ├── fd: (27)-->(28,29) + ├── project + │ ├── columns: cntrycode:27(string) c_acctbal:6(float!null) + │ ├── anti-join (merge) + │ │ ├── columns: c_custkey:1(int!null) c_phone:5(string!null) c_acctbal:6(float!null) + │ │ ├── left ordering: +1 + │ │ ├── right ordering: +19 + │ │ ├── key: (1) + │ │ ├── fd: (1)-->(5,6) + │ │ ├── select + │ │ │ ├── columns: c_custkey:1(int!null) c_phone:5(string!null) c_acctbal:6(float!null) + │ │ │ ├── key: (1) + │ │ │ ├── fd: (1)-->(5,6) + │ │ │ ├── ordering: +1 + │ │ │ ├── scan customer + │ │ │ │ ├── columns: c_custkey:1(int!null) c_phone:5(string!null) c_acctbal:6(float!null) + │ │ │ │ ├── key: (1) + │ │ │ │ ├── fd: (1)-->(5,6) + │ │ │ │ └── ordering: +1 + │ │ │ └── filters + │ │ │ ├── substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31') [type=bool, outer=(5)] + │ │ │ └── gt [type=bool, outer=(6), constraints=(/6: (/NULL - ])] + │ │ │ ├── variable: c_acctbal [type=float] + │ │ │ └── subquery [type=float] + │ │ │ └── scalar-group-by + │ │ │ ├── columns: avg:17(float) + │ │ │ ├── cardinality: [1 - 1] + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(17) + │ │ │ ├── select + │ │ │ │ ├── columns: c_phone:13(string!null) c_acctbal:14(float!null) + │ │ │ │ ├── scan customer + │ │ │ │ │ └── columns: c_phone:13(string!null) c_acctbal:14(float!null) + │ │ │ │ └── filters + │ │ │ │ ├── c_acctbal > 0.0 [type=bool, outer=(14), constraints=(/14: [/5e-324 - ]; tight)] + │ │ │ │ └── substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31') [type=bool, outer=(13)] + │ │ │ └── aggregations + │ │ │ └── avg [type=float, outer=(14)] + │ │ │ └── variable: c_acctbal [type=float] + │ │ ├── scan orders@o_ck + │ │ │ ├── columns: o_custkey:19(int!null) + │ │ │ └── ordering: +19 + │ │ └── filters (true) + │ └── projections + │ └── substring(c_phone, 1, 2) [type=string, outer=(5)] + └── aggregations + ├── count-rows [type=int] + └── sum [type=float, outer=(6)] + └── variable: c_acctbal [type=float] From d3aae3ef88eb1b08021d60fb63c0267bd7297201 Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Tue, 29 Jan 2019 12:16:38 +0100 Subject: [PATCH 04/10] storage: rename RaftTruncatedState -> LegacyRaftTruncatedState No functional changes, just preparing to introduce the shiny new unreplicated raft truncated state. Release note: None --- pkg/cli/debug.go | 2 +- pkg/cli/debug/print.go | 2 +- pkg/keys/constants.go | 4 ++-- pkg/keys/keys.go | 12 ++++++------ pkg/keys/keys_test.go | 2 +- pkg/keys/printer.go | 4 ++-- pkg/keys/printer_test.go | 2 +- pkg/storage/batcheval/cmd_truncate_log.go | 4 ++-- pkg/storage/rditer/replica_data_iter_test.go | 2 +- pkg/storage/replica_raftstorage.go | 6 +++--- pkg/storage/stateloader/stateloader.go | 20 ++++++++++---------- 11 files changed, 30 insertions(+), 30 deletions(-) diff --git a/pkg/cli/debug.go b/pkg/cli/debug.go index eefa7c54fb10..bc4f047e8895 100644 --- a/pkg/cli/debug.go +++ b/pkg/cli/debug.go @@ -665,7 +665,7 @@ func runDebugCheckStoreRaft(ctx context.Context, db *engine.RocksDB) error { return false, err } getReplicaInfo(rangeID).committedIndex = hs.Commit - case bytes.Equal(suffix, keys.LocalRaftTruncatedStateSuffix): + case bytes.Equal(suffix, keys.LocalRaftTruncatedStateLegacySuffix): var trunc roachpb.RaftTruncatedState if err := kv.Value.GetProto(&trunc); err != nil { return false, err diff --git a/pkg/cli/debug/print.go b/pkg/cli/debug/print.go index 18f02934b3e4..e5bcdc6b91ac 100644 --- a/pkg/cli/debug/print.go +++ b/pkg/cli/debug/print.go @@ -186,7 +186,7 @@ func tryRangeIDKey(kv engine.MVCCKeyValue) (string, error) { case bytes.Equal(suffix, keys.LocalRaftTombstoneSuffix): msg = &roachpb.RaftTombstone{} - case bytes.Equal(suffix, keys.LocalRaftTruncatedStateSuffix): + case bytes.Equal(suffix, keys.LocalRaftTruncatedStateLegacySuffix): msg = &roachpb.RaftTruncatedState{} case bytes.Equal(suffix, keys.LocalRangeLeaseSuffix): diff --git a/pkg/keys/constants.go b/pkg/keys/constants.go index b128db4fd434..ebee2fe9ea8a 100644 --- a/pkg/keys/constants.go +++ b/pkg/keys/constants.go @@ -121,8 +121,8 @@ var ( LocalRaftAppliedIndexLegacySuffix = []byte("rfta") // LocalRaftTombstoneSuffix is the suffix for the raft tombstone. LocalRaftTombstoneSuffix = []byte("rftb") - // LocalRaftTruncatedStateSuffix is the suffix for the RaftTruncatedState. - LocalRaftTruncatedStateSuffix = []byte("rftt") + // LocalRaftTruncatedStateLegacySuffix is the suffix for the RaftTruncatedState. + LocalRaftTruncatedStateLegacySuffix = []byte("rftt") // LocalRangeLeaseSuffix is the suffix for a range lease. LocalRangeLeaseSuffix = []byte("rll-") // LocalLeaseAppliedIndexLegacySuffix is the suffix for the applied lease index. diff --git a/pkg/keys/keys.go b/pkg/keys/keys.go index 1e05eb3761f5..b9af3cb223fe 100644 --- a/pkg/keys/keys.go +++ b/pkg/keys/keys.go @@ -252,9 +252,9 @@ func LeaseAppliedIndexLegacyKey(rangeID roachpb.RangeID) roachpb.Key { return MakeRangeIDPrefixBuf(rangeID).LeaseAppliedIndexLegacyKey() } -// RaftTruncatedStateKey returns a system-local key for a RaftTruncatedState. -func RaftTruncatedStateKey(rangeID roachpb.RangeID) roachpb.Key { - return MakeRangeIDPrefixBuf(rangeID).RaftTruncatedStateKey() +// RaftTruncatedStateLegacyKey returns a system-local key for a RaftTruncatedState. +func RaftTruncatedStateLegacyKey(rangeID roachpb.RangeID) roachpb.Key { + return MakeRangeIDPrefixBuf(rangeID).RaftTruncatedStateLegacyKey() } // RangeFrozenStatusKey returns a system-local key for the frozen status. @@ -897,9 +897,9 @@ func (b RangeIDPrefixBuf) LeaseAppliedIndexLegacyKey() roachpb.Key { return append(b.replicatedPrefix(), LocalLeaseAppliedIndexLegacySuffix...) } -// RaftTruncatedStateKey returns a system-local key for a RaftTruncatedState. -func (b RangeIDPrefixBuf) RaftTruncatedStateKey() roachpb.Key { - return append(b.replicatedPrefix(), LocalRaftTruncatedStateSuffix...) +// RaftTruncatedStateLegacyKey returns a system-local key for a RaftTruncatedState. +func (b RangeIDPrefixBuf) RaftTruncatedStateLegacyKey() roachpb.Key { + return append(b.replicatedPrefix(), LocalRaftTruncatedStateLegacySuffix...) } // RangeFrozenStatusKey returns a system-local key for the frozen status. diff --git a/pkg/keys/keys_test.go b/pkg/keys/keys_test.go index 8d282847f78f..090640d1cd56 100644 --- a/pkg/keys/keys_test.go +++ b/pkg/keys/keys_test.go @@ -146,7 +146,7 @@ func TestKeyAddressError(t *testing.T) { AbortSpanKey(0, uuid.MakeV4()), RaftTombstoneKey(0), RaftAppliedIndexLegacyKey(0), - RaftTruncatedStateKey(0), + RaftTruncatedStateLegacyKey(0), RangeLeaseKey(0), RangeStatsLegacyKey(0), RaftHardStateKey(0), diff --git a/pkg/keys/printer.go b/pkg/keys/printer.go index ebbbe36e05fe..af40f14d7c76 100644 --- a/pkg/keys/printer.go +++ b/pkg/keys/printer.go @@ -150,7 +150,7 @@ var ( ppFunc: raftLogKeyPrint, psFunc: raftLogKeyParse, }, - {name: "RaftTruncatedState", suffix: LocalRaftTruncatedStateSuffix}, + {name: "LegacyRaftTruncatedState", suffix: LocalRaftTruncatedStateLegacySuffix}, {name: "RaftLastIndex", suffix: LocalRaftLastIndexSuffix}, {name: "RangeLastReplicaGCTimestamp", suffix: LocalRangeLastReplicaGCTimestampSuffix}, {name: "RangeLastVerificationTimestamp", suffix: LocalRangeLastVerificationTimestampSuffixDeprecated}, @@ -575,7 +575,7 @@ func prettyPrintInternal(valDirs []encoding.Direction, key roachpb.Key, quoteRaw // /[rangeid]/RaftHardState "\x01s"+[rangeid]+"rfth" // /[rangeid]/RaftAppliedIndex "\x01s"+[rangeid]+"rfta" // /[rangeid]/RaftLog/logIndex:[logIndex] "\x01s"+[rangeid]+"rftl"+[logIndex] -// /[rangeid]/RaftTruncatedState "\x01s"+[rangeid]+"rftt" +// /[rangeid]/LegacyRaftTruncatedState "\x01s"+[rangeid]+"rftt" // /[rangeid]/RaftLastIndex "\x01s"+[rangeid]+"rfti" // /[rangeid]/RangeLastReplicaGCTimestamp "\x01s"+[rangeid]+"rlrt" // /[rangeid]/RangeLastVerificationTimestamp "\x01s"+[rangeid]+"rlvt" diff --git a/pkg/keys/printer_test.go b/pkg/keys/printer_test.go index b9cc118568bb..d9b449f39460 100644 --- a/pkg/keys/printer_test.go +++ b/pkg/keys/printer_test.go @@ -58,7 +58,7 @@ func TestPrettyPrint(t *testing.T) { {RangeAppliedStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeAppliedState"}, {RaftAppliedIndexLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftAppliedIndex"}, {LeaseAppliedIndexLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/LeaseAppliedIndex"}, - {RaftTruncatedStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTruncatedState"}, + {RaftTruncatedStateLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/LegacyRaftTruncatedState"}, {RangeLeaseKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeLease"}, {RangeStatsLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeStats"}, {RangeTxnSpanGCThresholdKey(roachpb.RangeID(1000001)), `/Local/RangeID/1000001/r/RangeTxnSpanGCThreshold`}, diff --git a/pkg/storage/batcheval/cmd_truncate_log.go b/pkg/storage/batcheval/cmd_truncate_log.go index b27c17dbf30f..26ea6d5092df 100644 --- a/pkg/storage/batcheval/cmd_truncate_log.go +++ b/pkg/storage/batcheval/cmd_truncate_log.go @@ -34,7 +34,7 @@ func init() { func declareKeysTruncateLog( _ roachpb.RangeDescriptor, header roachpb.Header, req roachpb.Request, spans *spanset.SpanSet, ) { - spans.Add(spanset.SpanReadWrite, roachpb.Span{Key: keys.RaftTruncatedStateKey(header.RangeID)}) + spans.Add(spanset.SpanReadWrite, roachpb.Span{Key: keys.RaftTruncatedStateLegacyKey(header.RangeID)}) prefix := keys.RaftLogPrefix(header.RangeID) spans.Add(spanset.SpanReadWrite, roachpb.Span{Key: prefix, EndKey: prefix.PrefixEnd()}) } @@ -111,5 +111,5 @@ func TruncateLog( } pd.Replicated.RaftLogDelta = ms.SysBytes - return pd, MakeStateLoader(cArgs.EvalCtx).SetTruncatedState(ctx, batch, cArgs.Stats, tState) + return pd, MakeStateLoader(cArgs.EvalCtx).SetLegacyRaftTruncatedState(ctx, batch, cArgs.Stats, tState) } diff --git a/pkg/storage/rditer/replica_data_iter_test.go b/pkg/storage/rditer/replica_data_iter_test.go index 4cda60b8857a..b1f53f44cc21 100644 --- a/pkg/storage/rditer/replica_data_iter_test.go +++ b/pkg/storage/rditer/replica_data_iter_test.go @@ -85,7 +85,7 @@ func createRangeData( {keys.RangeLastGCKey(desc.RangeID), ts0}, {keys.RangeAppliedStateKey(desc.RangeID), ts0}, {keys.RaftAppliedIndexLegacyKey(desc.RangeID), ts0}, - {keys.RaftTruncatedStateKey(desc.RangeID), ts0}, + {keys.RaftTruncatedStateLegacyKey(desc.RangeID), ts0}, {keys.RangeLeaseKey(desc.RangeID), ts0}, {keys.LeaseAppliedIndexLegacyKey(desc.RangeID), ts0}, {keys.RangeStatsLegacyKey(desc.RangeID), ts0}, diff --git a/pkg/storage/replica_raftstorage.go b/pkg/storage/replica_raftstorage.go index 74996fdae44b..4e1a6b88eb7c 100644 --- a/pkg/storage/replica_raftstorage.go +++ b/pkg/storage/replica_raftstorage.go @@ -212,7 +212,7 @@ func entries( } // No results, was it due to unavailability or truncation? - ts, err := rsl.LoadTruncatedState(ctx, e) + ts, err := rsl.LoadLegacyRaftTruncatedState(ctx, e) if err != nil { return nil, err } @@ -281,7 +281,7 @@ func term( // sideloaded entries. We only need the term, so this is what we do. ents, err := entries(ctx, rsl, eng, rangeID, eCache, nil /* sideloaded */, i, i+1, math.MaxUint64 /* maxBytes */) if err == raft.ErrCompacted { - ts, err := rsl.LoadTruncatedState(ctx, eng) + ts, err := rsl.LoadLegacyRaftTruncatedState(ctx, eng) if err != nil { return 0, err } @@ -318,7 +318,7 @@ func (r *Replica) raftTruncatedStateLocked( if r.mu.state.TruncatedState != nil { return *r.mu.state.TruncatedState, nil } - ts, err := r.mu.stateLoader.LoadTruncatedState(ctx, r.store.Engine()) + ts, err := r.mu.stateLoader.LoadLegacyRaftTruncatedState(ctx, r.store.Engine()) if err != nil { return ts, err } diff --git a/pkg/storage/stateloader/stateloader.go b/pkg/storage/stateloader/stateloader.go index 5b5fb360c360..995b19311136 100644 --- a/pkg/storage/stateloader/stateloader.go +++ b/pkg/storage/stateloader/stateloader.go @@ -105,7 +105,7 @@ func (rsl StateLoader) Load( // The truncated state should not be optional (i.e. the pointer is // pointless), but it is and the migration is not worth it. - truncState, err := rsl.LoadTruncatedState(ctx, reader) + truncState, err := rsl.LoadLegacyRaftTruncatedState(ctx, reader) if err != nil { return storagepb.ReplicaState{}, err } @@ -138,7 +138,7 @@ func (rsl StateLoader) Save( if err := rsl.SetTxnSpanGCThreshold(ctx, eng, ms, state.TxnSpanGCThreshold); err != nil { return enginepb.MVCCStats{}, err } - if err := rsl.SetTruncatedState(ctx, eng, ms, state.TruncatedState); err != nil { + if err := rsl.SetLegacyRaftTruncatedState(ctx, eng, ms, state.TruncatedState); err != nil { return enginepb.MVCCStats{}, err } if state.UsingAppliedStateKey { @@ -416,21 +416,21 @@ func (rsl StateLoader) SetMVCCStats( return rsl.writeLegacyMVCCStatsInternal(ctx, eng, newMS) } -// LoadTruncatedState loads the truncated state. -func (rsl StateLoader) LoadTruncatedState( +// LoadLegacyRaftTruncatedState loads the truncated state. +func (rsl StateLoader) LoadLegacyRaftTruncatedState( ctx context.Context, reader engine.Reader, ) (roachpb.RaftTruncatedState, error) { var truncState roachpb.RaftTruncatedState if _, err := engine.MVCCGetProto( - ctx, reader, rsl.RaftTruncatedStateKey(), hlc.Timestamp{}, &truncState, engine.MVCCGetOptions{}, + ctx, reader, rsl.RaftTruncatedStateLegacyKey(), hlc.Timestamp{}, &truncState, engine.MVCCGetOptions{}, ); err != nil { return roachpb.RaftTruncatedState{}, err } return truncState, nil } -// SetTruncatedState overwrites the truncated state. -func (rsl StateLoader) SetTruncatedState( +// SetLegacyRaftTruncatedState overwrites the truncated state. +func (rsl StateLoader) SetLegacyRaftTruncatedState( ctx context.Context, eng engine.ReadWriter, ms *enginepb.MVCCStats, @@ -440,7 +440,7 @@ func (rsl StateLoader) SetTruncatedState( return errors.New("cannot persist empty RaftTruncatedState") } return engine.MVCCPutProto(ctx, eng, ms, - rsl.RaftTruncatedStateKey(), hlc.Timestamp{}, nil, truncState) + rsl.RaftTruncatedStateLegacyKey(), hlc.Timestamp{}, nil, truncState) } // LoadGCThreshold loads the GC threshold. @@ -508,7 +508,7 @@ func (rsl StateLoader) LoadLastIndex(ctx context.Context, reader engine.Reader) if lastIndex == 0 { // The log is empty, which means we are either starting from scratch // or the entire log has been truncated away. - lastEnt, err := rsl.LoadTruncatedState(ctx, reader) + lastEnt, err := rsl.LoadLegacyRaftTruncatedState(ctx, reader) if err != nil { return 0, err } @@ -575,7 +575,7 @@ func (rsl StateLoader) SynthesizeRaftState(ctx context.Context, eng engine.ReadW if err != nil { return err } - truncState, err := rsl.LoadTruncatedState(ctx, eng) + truncState, err := rsl.LoadLegacyRaftTruncatedState(ctx, eng) if err != nil { return err } From 4ed9e5b00db715ad207cdd4186d36c4bf8a726e9 Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Tue, 29 Jan 2019 12:36:46 +0100 Subject: [PATCH 05/10] batcheval: add reminder when implementing divergent truncated states When replicas can have divergent truncated states, we want to carry out truncations even if they seem pointless to the leaseholder, since the leaseholder might have a shorter log than other replicas. Release note: None --- pkg/storage/batcheval/cmd_truncate_log.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/storage/batcheval/cmd_truncate_log.go b/pkg/storage/batcheval/cmd_truncate_log.go index 26ea6d5092df..ae61e3d53eb4 100644 --- a/pkg/storage/batcheval/cmd_truncate_log.go +++ b/pkg/storage/batcheval/cmd_truncate_log.go @@ -58,6 +58,7 @@ func TruncateLog( } // Have we already truncated this log? If so, just return without an error. + // TODO(tbg): remove this once followers can have divergent truncated states. firstIndex, err := cArgs.EvalCtx.GetFirstIndex() if err != nil { return result.Result{}, err From 2990b968c3c7aaae99a541fe0a6a8221602934ce Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Tue, 29 Jan 2019 12:59:14 +0100 Subject: [PATCH 06/10] keys: remove misleading overview over encoded keys The encoded range-local keys were mostly incorrect in that they were missing the replicated/unreplicated infix. Rather than trying to keep this comment up to date, readers should be directed to TestPrettyPrint which now conveniently logs all types of keys and their encoding. Release note: None --- pkg/keys/printer.go | 37 +++---------------------------------- pkg/keys/printer_test.go | 5 +++++ 2 files changed, 8 insertions(+), 34 deletions(-) diff --git a/pkg/keys/printer.go b/pkg/keys/printer.go index af40f14d7c76..33210ead7897 100644 --- a/pkg/keys/printer.go +++ b/pkg/keys/printer.go @@ -563,40 +563,9 @@ func prettyPrintInternal(valDirs []encoding.Direction, key roachpb.Key, quoteRaw return str } -// PrettyPrint prints the key in a human readable format: -// -// Key format Key value -// /Local/... "\x01"+... -// /Store/... "\x01s"+... -// /RangeID/... "\x01s"+[rangeid] -// /[rangeid]/AbortSpan/[id] "\x01s"+[rangeid]+"abc-"+[id] -// /[rangeid]/Lease "\x01s"+[rangeid]+"rfll" -// /[rangeid]/RaftTombstone "\x01s"+[rangeid]+"rftb" -// /[rangeid]/RaftHardState "\x01s"+[rangeid]+"rfth" -// /[rangeid]/RaftAppliedIndex "\x01s"+[rangeid]+"rfta" -// /[rangeid]/RaftLog/logIndex:[logIndex] "\x01s"+[rangeid]+"rftl"+[logIndex] -// /[rangeid]/LegacyRaftTruncatedState "\x01s"+[rangeid]+"rftt" -// /[rangeid]/RaftLastIndex "\x01s"+[rangeid]+"rfti" -// /[rangeid]/RangeLastReplicaGCTimestamp "\x01s"+[rangeid]+"rlrt" -// /[rangeid]/RangeLastVerificationTimestamp "\x01s"+[rangeid]+"rlvt" -// /[rangeid]/RangeStats "\x01s"+[rangeid]+"stat" -// /Range/... "\x01k"+... -// [key]/RangeDescriptor "\x01k"+[key]+"rdsc" -// [key]/Transaction/[id] "\x01k"+[key]+"txn-"+[txn-id] -// [key]/QueueLastProcessed/[queue] "\x01k"+[key]+"qlpt"+[queue] -// /Local/Max "\x02" -// -// /Meta1/[key] "\x02"+[key] -// /Meta2/[key] "\x03"+[key] -// /System/... "\x04" -// /NodeLiveness/[key] "\x04\0x00liveness-"+[key] -// /StatusNode/[key] "\x04status-node-"+[key] -// /System/Max "\x05" -// -// /Table/[key] [key] -// -// /Min "" -// /Max "\xff\xff" +// PrettyPrint prints the key in a human readable format, see TestPrettyPrint. +// The output does not indicate whether a key is part of the replicated or un- +// replicated keyspace. // // valDirs correspond to the encoding direction of each encoded value in key. // For example, table keys could have column values encoded in ascending or diff --git a/pkg/keys/printer_test.go b/pkg/keys/printer_test.go index d9b449f39460..4e3790be87cb 100644 --- a/pkg/keys/printer_test.go +++ b/pkg/keys/printer_test.go @@ -184,6 +184,11 @@ func TestPrettyPrint(t *testing.T) { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { keyInfo := MassagePrettyPrintedSpanForTest(PrettyPrint(nil /* valDirs */, test.key), nil) exp := MassagePrettyPrintedSpanForTest(test.exp, nil) + t.Logf(`---- test case #%d: +input: %q +output: %s +exp: %s +`, i+1, []byte(test.key), keyInfo, exp) if exp != keyInfo { t.Errorf("%d: expected %s, got %s", i, exp, keyInfo) } From 3878b12698dbbc422e007160dcb6712432c2c082 Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Wed, 30 Jan 2019 16:32:35 +0100 Subject: [PATCH 07/10] storage: add Store.VisitReplicas Release note: None --- pkg/storage/store.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/storage/store.go b/pkg/storage/store.go index ed2925e8e95e..c5497e144017 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -1788,6 +1788,13 @@ func (s *Store) GossipDeadReplicas(ctx context.Context) error { return s.cfg.Gossip.AddInfoProto(key, &deadReplicas, gossip.StoreTTL) } +// VisitReplicas invokes the visitor on the Store's Replicas until the visitor returns false. +// Replicas which are added to the Store after iteration begins may or may not be observed. +func (s *Store) VisitReplicas(visitor func(*Replica) bool) { + v := newStoreReplicaVisitor(s) + v.Visit(visitor) +} + // Bootstrap writes a new store ident to the underlying engine. To // ensure that no crufty data already exists in the engine, it scans // the engine contents before writing the new store ident. The engine From a132513613975c4b0bd74fa292bd8e8fc29e11f5 Mon Sep 17 00:00:00 2001 From: Yahor Yuzefovich Date: Sun, 10 Feb 2019 10:58:02 -0800 Subject: [PATCH 08/10] sql: reuse already allocated memory for the cache in a row container Previously, we would always allocate new memory for every row that we put in the cache of DiskBackedIndexedRowContainer and simply discard the memory underlying the row that we remove from the cache. Now, we're reusing that memory. Release note: None --- pkg/sql/rowcontainer/row_container.go | 104 ++++++++++++++++----- pkg/sql/rowcontainer/row_container_test.go | 104 ++++++++++++++++++--- 2 files changed, 173 insertions(+), 35 deletions(-) diff --git a/pkg/sql/rowcontainer/row_container.go b/pkg/sql/rowcontainer/row_container.go index 427bdc29d514..d85ab61ec7c9 100644 --- a/pkg/sql/rowcontainer/row_container.go +++ b/pkg/sql/rowcontainer/row_container.go @@ -534,8 +534,13 @@ type DiskBackedIndexedRowContainer struct { // [firstCachedRowPos, nextPosToCache). firstCachedRowPos int nextPosToCache int - indexedRowsCache ring.Buffer // the cache of up to maxIndexedRowsCacheSize contiguous rows - cacheMemAcc mon.BoundAccount + // indexedRowsCache is the cache of up to maxCacheSize contiguous rows. + indexedRowsCache ring.Buffer + // maxCacheSize indicates the maximum number of rows to be cached. It is + // initialized to maxIndexedRowsCacheSize and dynamically adjusted if OOM + // error is encountered. + maxCacheSize int + cacheMemAcc mon.BoundAccount } // MakeDiskBackedIndexedRowContainer creates a DiskBackedIndexedRowContainer @@ -570,6 +575,7 @@ func MakeDiskBackedIndexedRowContainer( d.scratchEncRow = make(sqlbase.EncDatumRow, len(d.storedTypes)) d.DiskBackedRowContainer = &DiskBackedRowContainer{} d.DiskBackedRowContainer.Init(ordering, d.storedTypes, evalCtx, engine, memoryMonitor, diskMonitor, rowCapacity) + d.maxCacheSize = maxIndexedRowsCacheSize d.cacheMemAcc = memoryMonitor.MakeBoundAccount() return &d } @@ -692,28 +698,38 @@ func (f *DiskBackedIndexedRowContainer) GetRow( } row, rowIdx := rowWithIdx[:len(rowWithIdx)-1], rowWithIdx[len(rowWithIdx)-1].Datum if idx, ok := rowIdx.(*tree.DInt); ok { - if f.indexedRowsCache.Len() == maxIndexedRowsCacheSize { - // The cache size is capped at maxIndexedRowsCacheSize, so we first - // remove the row with the smallest pos and advance + if f.indexedRowsCache.Len() == f.maxCacheSize { + // The cache size is capped at f.maxCacheSize, so we reuse the row + // with the smallest pos, put it as the last row, and advance // f.firstCachedRowPos. - usage := sizeOfInt + int64(f.indexedRowsCache.GetFirst().(IndexedRow).Row.Size()) - // TODO(yuzefovich): extend ring.Buffer to allow for reusing the - // allocated memory instead of allocating new one for every row. - f.indexedRowsCache.RemoveFirst() - // TODO(yuzefovich): investigate whether the pattern of growing and - // shrinking the memory account can be optimized. - f.cacheMemAcc.Shrink(ctx, usage) - f.firstCachedRowPos++ + if err := f.reuseFirstRowInCache(ctx, int(*idx), row); err != nil { + return nil, err + } + } else { + // We choose to ignore minor details like IndexedRow overhead and + // the cache overhead. + usage := sizeOfInt + int64(row.Size()) + if err := f.cacheMemAcc.Grow(ctx, usage); err != nil { + if sqlbase.IsOutOfMemoryError(err) { + // We hit the memory limit, so we need to cap the cache size + // and reuse the memory underlying first row in the cache. + if f.indexedRowsCache.Len() == 0 { + // The cache is empty, so there is no memory to be reused. + return nil, err + } + f.maxCacheSize = f.indexedRowsCache.Len() + if err := f.reuseFirstRowInCache(ctx, int(*idx), row); err != nil { + return nil, err + } + } else { + return nil, err + } + } else { + // We actually need to copy the row into memory. + ir := IndexedRow{int(*idx), f.rowAlloc.CopyRow(row)} + f.indexedRowsCache.AddLast(ir) + } } - // We choose to ignore minor details like IndexedRow overhead and - // the cache overhead. - usage := sizeOfInt + int64(row.Size()) - if err := f.cacheMemAcc.Grow(ctx, usage); err != nil { - return nil, err - } - // We actually need to copy the row into memory. - ir := IndexedRow{int(*idx), f.rowAlloc.CopyRow(row)} - f.indexedRowsCache.AddLast(ir) f.nextPosToCache++ } else { return nil, errors.Errorf("unexpected last column type: should be DInt but found %T", idx) @@ -733,6 +749,50 @@ func (f *DiskBackedIndexedRowContainer) GetRow( return nil, errors.Errorf("unexpected last column type: should be DInt but found %T", rowIdx) } +// reuseFirstRowInCache reuses the underlying memory of the first row in the +// cache to store 'row' and puts it as the last one in the cache. It adjusts +// the memory account accordingly and, if necessary, removes some first rows. +func (f *DiskBackedIndexedRowContainer) reuseFirstRowInCache( + ctx context.Context, idx int, row sqlbase.EncDatumRow, +) error { + newRowSize := row.Size() + for { + if f.indexedRowsCache.Len() == 0 { + return errors.Errorf("unexpectedly the cache of DiskBackedIndexedRowContainer contains zero rows") + } + indexedRowToReuse := f.indexedRowsCache.GetFirst().(IndexedRow) + oldRowSize := indexedRowToReuse.Row.Size() + delta := int64(newRowSize - oldRowSize) + if delta > 0 { + // New row takes up more memory than the old one. + if err := f.cacheMemAcc.Grow(ctx, delta); err != nil { + if sqlbase.IsOutOfMemoryError(err) { + // We need to actually reduce the cache size, so we remove the first + // row and adjust the memory account, maxCacheSize, and + // f.firstCachedRowPos accordingly. + f.indexedRowsCache.RemoveFirst() + f.cacheMemAcc.Shrink(ctx, int64(oldRowSize)) + f.maxCacheSize-- + f.firstCachedRowPos++ + if f.indexedRowsCache.Len() == 0 { + return err + } + continue + } + return err + } + } else if delta < 0 { + f.cacheMemAcc.Shrink(ctx, -delta) + } + indexedRowToReuse.Idx = idx + copy(indexedRowToReuse.Row, row) + f.indexedRowsCache.RemoveFirst() + f.indexedRowsCache.AddLast(indexedRowToReuse) + f.firstCachedRowPos++ + return nil + } +} + // getRowWithoutCache returns the row at requested position without using the // cache. It utilizes the same disk row iterator along multiple consequent // calls and rewinds the iterator only when it has been advanced further than diff --git a/pkg/sql/rowcontainer/row_container_test.go b/pkg/sql/rowcontainer/row_container_test.go index 8db929b6a8ef..840a82712a30 100644 --- a/pkg/sql/rowcontainer/row_container_test.go +++ b/pkg/sql/rowcontainer/row_container_test.go @@ -316,6 +316,7 @@ func TestDiskBackedRowContainer(t *testing.T) { memoryMonitor.Start(ctx, nil, mon.MakeStandaloneBudget(1)) defer memoryMonitor.Stop(ctx) diskMonitor.Start(ctx, nil, mon.MakeStandaloneBudget(1)) + defer diskMonitor.Stop(ctx) defer func() { if err := rc.UnsafeReset(ctx); err != nil { @@ -411,7 +412,6 @@ func TestDiskBackedIndexedRowContainer(t *testing.T) { const numTestRuns = 10 const numRows = 10 const numCols = 2 - rows := make([]sqlbase.EncDatumRow, numRows) ordering := sqlbase.ColumnOrdering{{ColIdx: 0, Direction: encoding.Ascending}} newOrdering := sqlbase.ColumnOrdering{{ColIdx: 1, Direction: encoding.Ascending}} @@ -427,6 +427,7 @@ func TestDiskBackedIndexedRowContainer(t *testing.T) { // index). t.Run("SpillingHalfway", func(t *testing.T) { for i := 0; i < numTestRuns; i++ { + rows := make([]sqlbase.EncDatumRow, numRows) types := sqlbase.RandSortingColumnTypes(rng, numCols) for i := 0; i < numRows; i++ { rows[i] = sqlbase.RandEncDatumRowOfTypes(rng, types) @@ -482,6 +483,7 @@ func TestDiskBackedIndexedRowContainer(t *testing.T) { // to be returned. Then, it spills to disk and does the same check again. t.Run("TestGetRow", func(t *testing.T) { for i := 0; i < numTestRuns; i++ { + rows := make([]sqlbase.EncDatumRow, numRows) sortedRows := indexedRows{rows: make([]IndexedRow, numRows)} types := sqlbase.RandSortingColumnTypes(rng, numCols) for i := 0; i < numRows; i++ { @@ -489,7 +491,7 @@ func TestDiskBackedIndexedRowContainer(t *testing.T) { sortedRows.rows[i] = IndexedRow{Idx: i, Row: rows[i]} } - sorter := sorter{evalCtx: &evalCtx, rows: sortedRows, ordering: ordering} + sorter := rowsSorter{evalCtx: &evalCtx, rows: sortedRows, ordering: ordering} sort.Sort(&sorter) if sorter.err != nil { t.Fatal(sorter.err) @@ -559,6 +561,86 @@ func TestDiskBackedIndexedRowContainer(t *testing.T) { } }) + // TestGetRowFromDiskWithLimitedMemory forces the container to spill to disk, + // adds all rows to it, sorts them, and checks that both the index and the + // row are what we expect by GetRow() to be returned. The goal is to test the + // behavior of capping the cache size and reusing the memory of the first + // rows in the cache, so we use the memory budget that accommodates only + // about half of all rows in the cache. + t.Run("TestGetRowWithLimitedMemory", func(t *testing.T) { + for i := 0; i < numTestRuns; i++ { + budget := int64(10240) + memoryUsage := int64(0) + rows := make([]sqlbase.EncDatumRow, 0, numRows) + sortedRows := indexedRows{rows: make([]IndexedRow, 0, numRows)} + types := sqlbase.RandSortingColumnTypes(rng, numCols) + for memoryUsage < 2*budget { + row := sqlbase.RandEncDatumRowOfTypes(rng, types) + memoryUsage += int64(row.Size()) + rows = append(rows, row) + sortedRows.rows = append(sortedRows.rows, IndexedRow{Idx: len(sortedRows.rows), Row: row}) + } + + memoryMonitor.Start(ctx, nil, mon.MakeStandaloneBudget(budget)) + defer memoryMonitor.Stop(ctx) + diskMonitor.Start(ctx, nil, mon.MakeStandaloneBudget(math.MaxInt64)) + defer diskMonitor.Stop(ctx) + + sorter := rowsSorter{evalCtx: &evalCtx, rows: sortedRows, ordering: ordering} + sort.Sort(&sorter) + if sorter.err != nil { + t.Fatal(sorter.err) + } + + func() { + rc := MakeDiskBackedIndexedRowContainer(ordering, types, &evalCtx, tempEngine, &memoryMonitor, &diskMonitor, 0 /* rowCapacity */) + defer rc.Close(ctx) + if err := rc.spillToDisk(ctx); err != nil { + t.Fatal(err) + } + for _, row := range rows { + if err := rc.AddRow(ctx, row); err != nil { + t.Fatal(err) + } + } + if !rc.Spilled() { + t.Fatal("unexpectedly using memory") + } + rc.Sort(ctx) + + // Check that GetRow returns the row we expect at each position. + for i := 0; i < len(rows); i++ { + readRow, err := rc.GetRow(ctx, i) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + expectedRow := sortedRows.rows[i] + readOrderingDatum, err := readRow.GetDatum(ordering[0].ColIdx) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if readOrderingDatum.Compare(&evalCtx, expectedRow.Row[ordering[0].ColIdx].Datum) != 0 { + // We're skipping comparison if both rows are equal on the ordering + // column since in this case the order of indexed rows after + // sorting is nondeterministic. + if readRow.GetIdx() != expectedRow.GetIdx() { + t.Fatalf("read row has different idx that what we expect") + } + for col, expectedDatum := range expectedRow.Row { + readDatum, err := readRow.GetDatum(col) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cmp := readDatum.Compare(&evalCtx, expectedDatum.Datum); cmp != 0 { + t.Fatalf("read row is not equal to expected one") + } + } + } + } + }() + } + }) + // ReorderingInMemory initializes a DiskBackedIndexedRowContainer with one // ordering, adds all rows to it, sorts it and makes sure that the rows are // sorted as expected. Then, it reorders the container to a different @@ -566,6 +648,7 @@ func TestDiskBackedIndexedRowContainer(t *testing.T) { // Only in-memory containers should be used. t.Run("ReorderingInMemory", func(t *testing.T) { for i := 0; i < numTestRuns; i++ { + rows := make([]sqlbase.EncDatumRow, numRows) types := sqlbase.RandSortingColumnTypes(rng, numCols) for i := 0; i < numRows; i++ { rows[i] = sqlbase.RandEncDatumRowOfTypes(rng, types) @@ -606,6 +689,7 @@ func TestDiskBackedIndexedRowContainer(t *testing.T) { // container is forced to spill to disk right after initialization. t.Run("ReorderingOnDisk", func(t *testing.T) { for i := 0; i < numTestRuns; i++ { + rows := make([]sqlbase.EncDatumRow, numRows) types := sqlbase.RandSortingColumnTypes(rng, numCols) for i := 0; i < numRows; i++ { rows[i] = sqlbase.RandEncDatumRowOfTypes(rng, types) @@ -659,19 +743,19 @@ func (ir indexedRows) Len() int { // TODO(yuzefovich): this is a duplicate of partitionSorter from windower.go. // There are possibly couple of other duplicates as well in other files, so we // should refactor it and probably extract the code into a new package. -type sorter struct { +type rowsSorter struct { evalCtx *tree.EvalContext rows indexedRows ordering sqlbase.ColumnOrdering err error } -func (n *sorter) Len() int { return n.rows.Len() } +func (n *rowsSorter) Len() int { return n.rows.Len() } -func (n *sorter) Swap(i, j int) { +func (n *rowsSorter) Swap(i, j int) { n.rows.rows[i], n.rows.rows[j] = n.rows.rows[j], n.rows.rows[i] } -func (n *sorter) Less(i, j int) bool { +func (n *rowsSorter) Less(i, j int) bool { if n.err != nil { // An error occurred in previous calls to Less(). We want to be done with // sorting and to propagate that error to the caller of Sort(). @@ -685,13 +769,7 @@ func (n *sorter) Less(i, j int) bool { return cmp < 0 } -// sorter implements the tree.PeerGroupChecker interface. -func (n *sorter) InSameGroup(i, j int) (bool, error) { - cmp, err := n.Compare(i, j) - return cmp == 0, err -} - -func (n *sorter) Compare(i, j int) (int, error) { +func (n *rowsSorter) Compare(i, j int) (int, error) { ra, rb := n.rows.rows[i], n.rows.rows[j] for _, o := range n.ordering { da, err := ra.GetDatum(o.ColIdx) From d0aa09e6afc802ee9d02f367925220c0e1f9a0aa Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Mon, 11 Feb 2019 22:26:45 +0100 Subject: [PATCH 09/10] storage: make RaftTruncatedState unreplicated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See #34287. Today, Raft (or preemptive) snapshots include the past Raft log, that is, log entries which are already reflected in the state of the snapshot. Fundamentally, this is because we have historically used a replicated TruncatedState. TruncatedState essentially tells us what the first index in the log is (though it also includes a Term). If the TruncatedState cannot diverge across replicas, we *must* send the whole log in snapshots, as the first log index must match what the TruncatedState claims it is. The Raft log is typically, but not necessarily small. Log truncations are driven by a queue and use a complex decision process. That decision process can be faulty and even if it isn't, the queue could be held up. Besides, even when the Raft log contains only very few entries, these entries may be quite large (see SSTable ingestion during RESTORE). All this motivates that we don't want to (be forced to) send the Raft log as part of snapshots, and in turn we need the TruncatedState to be unreplicated. This change migrates the TruncatedState into unreplicated keyspace. It does not yet allow snapshots to avoid sending the past Raft log, but that is a relatively straightforward follow-up change. VersionUnreplicatedRaftTruncatedState, when active, moves the truncated state into unreplicated keyspace on log truncations. The migration works as follows: 1. at any log position, the replicas of a Range either use the new (unreplicated) key or the old one, and exactly one of them exists. 2. When a log truncation evaluates under the new cluster version, it initiates the migration by deleting the old key. Under the old cluster version, it behaves like today, updating the replicated truncated state. 3. The deletion signals new code downstream of Raft and triggers a write to the new, unreplicated, key (atomic with the deletion of the old key). 4. Future log truncations don't write any replicated data any more, but (like before) send along the TruncatedState which is written downstream of Raft atomically with the deletion of the log entries. This actually uses the same code as 3. What's new is that the truncated state needs to be verified before replacing a previous one. If replicas disagree about their truncated state, it's possible for replica X at FirstIndex=100 to apply a truncated state update that sets FirstIndex to, say, 50 (proposed by a replica with a "longer" historical log). In that case, the truncated state update must be ignored (this is straightforward downstream-of-Raft code). 5. When a split trigger evaluates, it seeds the RHS with the legacy key iff the LHS uses the legacy key, and the unreplicated key otherwise. This makes sure that the invariant that all replicas agree on the state of the migration is upheld. 6. When a snapshot is applied, the receiver is told whether the snapshot contains a legacy key. If not, it writes the truncated state (which is part of the snapshot metadata) in its unreplicated version. Otherwise it doesn't have to do anything (the range will migrate later). The following diagram visualizes the above. Note that it abuses sequence diagrams to get a nice layout; the vertical lines belonging to NewState and OldState don't imply any particular ordering of operations. ``` ┌────────┐ ┌────────┐ │OldState│ │NewState│ └───┬────┘ └───┬────┘ │ Bootstrap under old version │ <─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ │ │ │ Bootstrap under new version │ │ <─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ │ │─ ─ ┐ │ | Log truncation under old version │< ─ ┘ │ │ │─ ─ ┐ │ │ | Snapshot │ │< ─ ┘ │ │ │ │ │─ ─ ┐ │ │ | Snapshot │ │< ─ ┘ │ │ │ Log truncation under new version │ │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─>│ │ │ │ │─ ─ ┐ │ │ | Log truncation under new version │ │< ─ ┘ │ │ │ │─ ─ ┐ │ │ | Log truncation under old version │ │< ─ ┘ (necessarily running new binary) ``` Source: http://www.plantuml.com/plantuml/uml/ and the following input: @startuml scale 600 width OldState <--] : Bootstrap under old version NewState <--] : Bootstrap under new version OldState --> OldState : Log truncation under old version OldState --> OldState : Snapshot NewState --> NewState : Snapshot OldState --> NewState : Log truncation under new version NewState --> NewState : Log truncation under new version NewState --> NewState : Log truncation under old version\n(necessarily running new binary) @enduml Release note: None --- docs/generated/settings/settings.html | 2 +- pkg/keys/constants.go | 3 +- pkg/keys/keys.go | 11 + pkg/keys/printer.go | 2 +- pkg/keys/printer_test.go | 3 +- pkg/server/version_cluster_test.go | 172 ++++++++++++++ pkg/settings/cluster/cockroach_versions.go | 93 ++++++++ .../testdata/logic_test/crdb_internal | 4 +- pkg/storage/batcheval/cmd_end_transaction.go | 24 ++ .../batcheval/cmd_resolve_intent_test.go | 19 +- pkg/storage/batcheval/cmd_truncate_log.go | 55 ++++- pkg/storage/batcheval/truncate_log_test.go | 200 ++++++++++++++++ pkg/storage/below_raft_protos_test.go | 11 + pkg/storage/engine/engine.go | 10 +- pkg/storage/main_test.go | 2 +- pkg/storage/raft.pb.go | 223 +++++++++++------- pkg/storage/raft.proto | 11 + pkg/storage/replica_command.go | 22 ++ pkg/storage/replica_raft.go | 135 ++++++++--- pkg/storage/replica_raft_truncation_test.go | 119 ++++++++++ pkg/storage/replica_raftstorage.go | 32 ++- pkg/storage/replica_test.go | 6 +- pkg/storage/stateloader/initial.go | 11 +- pkg/storage/stateloader/stateloader.go | 87 +++++-- pkg/storage/stats_test.go | 4 +- pkg/storage/store.go | 15 +- pkg/storage/store_snapshot.go | 11 +- pkg/storage/store_test.go | 2 + .../truncated_state_migration/migration | 25 ++ .../truncated_state_migration/pre_migration | 26 ++ 30 files changed, 1165 insertions(+), 175 deletions(-) create mode 100644 pkg/storage/batcheval/truncate_log_test.go create mode 100644 pkg/storage/replica_raft_truncation_test.go create mode 100644 pkg/storage/testdata/truncated_state_migration/migration create mode 100644 pkg/storage/testdata/truncated_state_migration/pre_migration diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html index d8f8c6ba6121..914d334755a5 100644 --- a/docs/generated/settings/settings.html +++ b/docs/generated/settings/settings.html @@ -95,6 +95,6 @@ trace.debug.enablebooleanfalseif set, traces for recent requests can be seen in the /debug page trace.lightstep.tokenstringif set, traces go to Lightstep using this token trace.zipkin.collectorstringif set, traces go to the given Zipkin instance (example: '127.0.0.1:9411'); ignored if trace.lightstep.token is set. -versioncustom validation2.1-5set the active cluster version in the format '.'. +versioncustom validation2.1-6set the active cluster version in the format '.'. diff --git a/pkg/keys/constants.go b/pkg/keys/constants.go index ebee2fe9ea8a..6bd5e0bcfcf3 100644 --- a/pkg/keys/constants.go +++ b/pkg/keys/constants.go @@ -121,7 +121,8 @@ var ( LocalRaftAppliedIndexLegacySuffix = []byte("rfta") // LocalRaftTombstoneSuffix is the suffix for the raft tombstone. LocalRaftTombstoneSuffix = []byte("rftb") - // LocalRaftTruncatedStateLegacySuffix is the suffix for the RaftTruncatedState. + // LocalRaftTruncatedStateLegacySuffix is the suffix for the legacy RaftTruncatedState. + // See VersionUnreplicatedRaftTruncatedState. LocalRaftTruncatedStateLegacySuffix = []byte("rftt") // LocalRangeLeaseSuffix is the suffix for a range lease. LocalRangeLeaseSuffix = []byte("rll-") diff --git a/pkg/keys/keys.go b/pkg/keys/keys.go index b9af3cb223fe..8d0d1e91f8f0 100644 --- a/pkg/keys/keys.go +++ b/pkg/keys/keys.go @@ -253,6 +253,7 @@ func LeaseAppliedIndexLegacyKey(rangeID roachpb.RangeID) roachpb.Key { } // RaftTruncatedStateLegacyKey returns a system-local key for a RaftTruncatedState. +// See VersionUnreplicatedRaftTruncatedState. func RaftTruncatedStateLegacyKey(rangeID roachpb.RangeID) roachpb.Key { return MakeRangeIDPrefixBuf(rangeID).RaftTruncatedStateLegacyKey() } @@ -314,6 +315,11 @@ func RaftTombstoneKey(rangeID roachpb.RangeID) roachpb.Key { return MakeRangeIDPrefixBuf(rangeID).RaftTombstoneKey() } +// RaftTruncatedStateKey returns a system-local key for a RaftTruncatedState. +func RaftTruncatedStateKey(rangeID roachpb.RangeID) roachpb.Key { + return MakeRangeIDPrefixBuf(rangeID).RaftTruncatedStateKey() +} + // RaftHardStateKey returns a system-local key for a Raft HardState. func RaftHardStateKey(rangeID roachpb.RangeID) roachpb.Key { return MakeRangeIDPrefixBuf(rangeID).RaftHardStateKey() @@ -935,6 +941,11 @@ func (b RangeIDPrefixBuf) RaftTombstoneKey() roachpb.Key { return append(b.unreplicatedPrefix(), LocalRaftTombstoneSuffix...) } +// RaftTruncatedStateKey returns a system-local key for a RaftTruncatedState. +func (b RangeIDPrefixBuf) RaftTruncatedStateKey() roachpb.Key { + return append(b.unreplicatedPrefix(), LocalRaftTruncatedStateLegacySuffix...) +} + // RaftHardStateKey returns a system-local key for a Raft HardState. func (b RangeIDPrefixBuf) RaftHardStateKey() roachpb.Key { return append(b.unreplicatedPrefix(), LocalRaftHardStateSuffix...) diff --git a/pkg/keys/printer.go b/pkg/keys/printer.go index 33210ead7897..3fe3a46e1c15 100644 --- a/pkg/keys/printer.go +++ b/pkg/keys/printer.go @@ -150,7 +150,7 @@ var ( ppFunc: raftLogKeyPrint, psFunc: raftLogKeyParse, }, - {name: "LegacyRaftTruncatedState", suffix: LocalRaftTruncatedStateLegacySuffix}, + {name: "RaftTruncatedState", suffix: LocalRaftTruncatedStateLegacySuffix}, {name: "RaftLastIndex", suffix: LocalRaftLastIndexSuffix}, {name: "RangeLastReplicaGCTimestamp", suffix: LocalRangeLastReplicaGCTimestampSuffix}, {name: "RangeLastVerificationTimestamp", suffix: LocalRangeLastVerificationTimestampSuffixDeprecated}, diff --git a/pkg/keys/printer_test.go b/pkg/keys/printer_test.go index 4e3790be87cb..f3b679766d2e 100644 --- a/pkg/keys/printer_test.go +++ b/pkg/keys/printer_test.go @@ -58,7 +58,8 @@ func TestPrettyPrint(t *testing.T) { {RangeAppliedStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeAppliedState"}, {RaftAppliedIndexLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftAppliedIndex"}, {LeaseAppliedIndexLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/LeaseAppliedIndex"}, - {RaftTruncatedStateLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/LegacyRaftTruncatedState"}, + {RaftTruncatedStateLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RaftTruncatedState"}, + {RaftTruncatedStateKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/u/RaftTruncatedState"}, {RangeLeaseKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeLease"}, {RangeStatsLegacyKey(roachpb.RangeID(1000001)), "/Local/RangeID/1000001/r/RangeStats"}, {RangeTxnSpanGCThresholdKey(roachpb.RangeID(1000001)), `/Local/RangeID/1000001/r/RangeTxnSpanGCThreshold`}, diff --git a/pkg/server/version_cluster_test.go b/pkg/server/version_cluster_test.go index afca6ccfd360..bc00d92ef0cb 100644 --- a/pkg/server/version_cluster_test.go +++ b/pkg/server/version_cluster_test.go @@ -24,16 +24,20 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" + "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" ) type testClusterWithHelpers struct { @@ -191,6 +195,174 @@ func TestClusterVersionPersistedOnJoin(t *testing.T) { } } +// TestClusterVersionUnreplicatedRaftTruncatedState exercises the +// VersionUnreplicatedRaftTruncatedState migration in as much detail as possible +// in a unit test. +// +// It starts a four node cluster with a pre-migration version and upgrades into +// the new version while traffic and scattering are active, verifying that the +// truncated states are rewritten. +func TestClusterVersionUnreplicatedRaftTruncatedState(t *testing.T) { + defer leaktest.AfterTest(t)() + ctx := context.Background() + + dir, finish := testutils.TempDir(t) + defer finish() + + oldVersion := cluster.VersionByKey(cluster.VersionUnreplicatedRaftTruncatedState - 1) + oldVersionS := oldVersion.String() + newVersionS := cluster.VersionByKey(cluster.VersionUnreplicatedRaftTruncatedState).String() + + // Four node cluster in which all versions support newVersion (i.e. would in + // principle upgrade to it) but are bootstrapped at oldVersion. + versions := [][2]string{ + {oldVersionS, newVersionS}, + {oldVersionS, newVersionS}, + {oldVersionS, newVersionS}, + {oldVersionS, newVersionS}, + } + + bootstrapVersion := cluster.ClusterVersion{Version: oldVersion} + + knobs := base.TestingKnobs{ + Store: &storage.StoreTestingKnobs{ + BootstrapVersion: &bootstrapVersion, + }, + Server: &server.TestingKnobs{ + DisableAutomaticVersionUpgrade: 1, + }, + } + + tc := setupMixedCluster(t, knobs, versions, dir) + defer tc.TestCluster.Stopper().Stop(ctx) + + if _, err := tc.ServerConn(0).Exec(` +CREATE TABLE kv (id INT PRIMARY KEY, v INT); +ALTER TABLE kv SPLIT AT SELECT i FROM generate_series(1, 9) AS g(i); +`); err != nil { + t.Fatal(err) + } + + scatter := func() { + t.Helper() + if _, err := tc.ServerConn(0).Exec( + `ALTER TABLE kv EXPERIMENTAL_RELOCATE SELECT ARRAY[i%$1+1], i FROM generate_series(0, 9) AS g(i)`, len(versions), + ); err != nil { + t.Log(err) + } + } + + var n int + insert := func() { + t.Helper() + n++ + // Write only to a subset of our ranges to guarantee log truncations there. + _, err := tc.ServerConn(0).Exec(`UPSERT INTO kv VALUES($1, $2)`, n%2, n) + if err != nil { + t.Fatal(err) + } + } + + for i := 0; i < 500; i++ { + insert() + } + scatter() + + for _, server := range tc.Servers { + assert.NoError(t, server.GetStores().(*storage.Stores).VisitStores(func(s *storage.Store) error { + s.VisitReplicas(func(r *storage.Replica) bool { + key := keys.RaftTruncatedStateKey(r.RangeID) + var truncState roachpb.RaftTruncatedState + found, err := engine.MVCCGetProto( + context.Background(), s.Engine(), key, + hlc.Timestamp{}, &truncState, engine.MVCCGetOptions{}, + ) + if err != nil { + t.Fatal(err) + } + if found { + t.Errorf("unexpectedly found unreplicated TruncatedState at %s", key) + } + return true // want more + }) + return nil + })) + } + + if v := tc.getVersionFromSelect(0); v != oldVersionS { + t.Fatalf("running %s, wanted %s", v, oldVersionS) + } + + assert.NoError(t, tc.setVersion(0, newVersionS)) + for i := 0; i < 500; i++ { + insert() + } + scatter() + + for _, server := range tc.Servers { + testutils.SucceedsSoon(t, func() error { + err := server.GetStores().(*storage.Stores).VisitStores(func(s *storage.Store) error { + // We scattered and so old copies of replicas may be laying around. + // If we're not proactive about removing them, the test gets pretty + // slow because those replicas aren't caught up any more. + s.MustForceReplicaGCScanAndProcess() + var err error + s.VisitReplicas(func(r *storage.Replica) bool { + snap := s.Engine().NewSnapshot() + defer snap.Close() + + keyLegacy := keys.RaftTruncatedStateLegacyKey(r.RangeID) + keyUnreplicated := keys.RaftTruncatedStateKey(r.RangeID) + + if found, innerErr := engine.MVCCGetProto( + context.Background(), snap, keyLegacy, + hlc.Timestamp{}, nil, engine.MVCCGetOptions{}, + ); innerErr != nil { + t.Fatal(innerErr) + } else if found { + if err == nil { + err = errors.New("found legacy TruncatedState") + } + err = errors.Wrap(err, r.String()) + + // Force a log truncation to prove that this rectifies + // the situation. + status := r.RaftStatus() + if status != nil { + desc := r.Desc() + truncate := &roachpb.TruncateLogRequest{} + truncate.Key = desc.StartKey.AsRawKey() + truncate.RangeID = desc.RangeID + truncate.Index = status.HardState.Commit + var ba roachpb.BatchRequest + ba.RangeID = r.RangeID + ba.Add(truncate) + if _, err := s.DB().NonTransactionalSender().Send(ctx, ba); err != nil { + t.Fatal(err) + } + } + return true // want more + } + + if found, err := engine.MVCCGetProto( + context.Background(), snap, keyUnreplicated, + hlc.Timestamp{}, nil, engine.MVCCGetOptions{}, + ); err != nil { + t.Fatal(err) + } else if !found { + // We can't have neither of the keys present. + t.Fatalf("%s: unexpectedly did not find unreplicated TruncatedState at %s", r, keyUnreplicated) + } + + return true // want more + }) + return err + }) + return err + }) + } +} + func TestClusterVersionUpgrade(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() diff --git a/pkg/settings/cluster/cockroach_versions.go b/pkg/settings/cluster/cockroach_versions.go index cd151f156b2a..5601ca32d40e 100644 --- a/pkg/settings/cluster/cockroach_versions.go +++ b/pkg/settings/cluster/cockroach_versions.go @@ -71,6 +71,7 @@ const ( VersionExportStorageWorkload VersionLazyTxnRecord VersionSequencedReads + VersionUnreplicatedRaftTruncatedState // see versionsSingleton for details // Add new versions here (step one of two). @@ -315,6 +316,98 @@ var versionsSingleton = keyedVersions([]keyedVersion{ Key: VersionSequencedReads, Version: roachpb.Version{Major: 2, Minor: 1, Unstable: 5}, }, + { + // VersionLazyTxnRecord is https://github.com/cockroachdb/cockroach/pull/34660. + // When active, it moves the truncated state into unreplicated keyspace + // on log truncations. + // + // The migration works as follows: + // + // 1. at any log position, the replicas of a Range either use the new + // (unreplicated) key or the old one, and exactly one of them exists. + // + // 2. When a log truncation evaluates under the new cluster version, + // it initiates the migration by deleting the old key. Under the old cluster + // version, it behaves like today, updating the replicated truncated state. + // + // 3. The deletion signals new code downstream of Raft and triggers a write + // to the new, unreplicated, key (atomic with the deletion of the old key). + // + // 4. Future log truncations don't write any replicated data any more, but + // (like before) send along the TruncatedState which is written downstream + // of Raft atomically with the deletion of the log entries. This actually + // uses the same code as 3. + // What's new is that the truncated state needs to be verified before + // replacing a previous one. If replicas disagree about their truncated + // state, it's possible for replica X at FirstIndex=100 to apply a + // truncated state update that sets FirstIndex to, say, 50 (proposed by a + // replica with a "longer" historical log). In that case, the truncated + // state update must be ignored (this is straightforward downstream-of-Raft + // code). + // + // 5. When a split trigger evaluates, it seeds the RHS with the legacy + // key iff the LHS uses the legacy key, and the unreplicated key otherwise. + // This makes sure that the invariant that all replicas agree on the + // state of the migration is upheld. + // + // 6. When a snapshot is applied, the receiver is told whether the snapshot + // contains a legacy key. If not, it writes the truncated state (which is + // part of the snapshot metadata) in its unreplicated version. Otherwise + // it doesn't have to do anything (the range will migrate later). + // + // The following diagram visualizes the above. Note that it abuses sequence + // diagrams to get a nice layout; the vertical lines belonging to NewState + // and OldState don't imply any particular ordering of operations. + // + // ┌────────┐ ┌────────┐ + // │OldState│ │NewState│ + // └───┬────┘ └───┬────┘ + // │ Bootstrap under old version + // │ <─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ + // │ │ + // │ │ Bootstrap under new version + // │ │ <─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ + // │ │ + // │─ ─ ┐ + // │ | Log truncation under old version + // │< ─ ┘ + // │ │ + // │─ ─ ┐ │ + // │ | Snapshot │ + // │< ─ ┘ │ + // │ │ + // │ │─ ─ ┐ + // │ │ | Snapshot + // │ │< ─ ┘ + // │ │ + // │ Log truncation under new version │ + // │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─>│ + // │ │ + // │ │─ ─ ┐ + // │ │ | Log truncation under new version + // │ │< ─ ┘ + // │ │ + // │ │─ ─ ┐ + // │ │ | Log truncation under old version + // │ │< ─ ┘ (necessarily running new binary) + // + // Source: http://www.plantuml.com/plantuml/uml/ and the following input: + // + // @startuml + // scale 600 width + // + // OldState <--] : Bootstrap under old version + // NewState <--] : Bootstrap under new version + // OldState --> OldState : Log truncation under old version + // OldState --> OldState : Snapshot + // NewState --> NewState : Snapshot + // OldState --> NewState : Log truncation under new version + // NewState --> NewState : Log truncation under new version + // NewState --> NewState : Log truncation under old version\n(necessarily running new binary) + // @enduml + Key: VersionUnreplicatedRaftTruncatedState, + Version: roachpb.Version{Major: 2, Minor: 1, Unstable: 6}, + }, // Add new versions here (step two of two). diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal b/pkg/sql/logictest/testdata/logic_test/crdb_internal index c035e310449e..7f956f1f4ea5 100644 --- a/pkg/sql/logictest/testdata/logic_test/crdb_internal +++ b/pkg/sql/logictest/testdata/logic_test/crdb_internal @@ -268,7 +268,7 @@ select crdb_internal.set_vmodule('') query T select crdb_internal.node_executable_version() ---- -2.1-5 +2.1-6 query ITTT colnames select node_id, component, field, regexp_replace(regexp_replace(value, '^\d+$', ''), e':\\d+', ':') as value from crdb_internal.node_runtime_info @@ -365,7 +365,7 @@ select * from crdb_internal.gossip_alerts query T select crdb_internal.node_executable_version() ---- -2.1-5 +2.1-6 user root diff --git a/pkg/storage/batcheval/cmd_end_transaction.go b/pkg/storage/batcheval/cmd_end_transaction.go index 82a432c4884d..49fb8970c5c3 100644 --- a/pkg/storage/batcheval/cmd_end_transaction.go +++ b/pkg/storage/batcheval/cmd_end_transaction.go @@ -911,6 +911,29 @@ func splitTrigger( log.VEventf(ctx, 1, "LHS's TxnSpanGCThreshold of split is not set") } + // We're about to write the initial state for the replica. We migrated + // the formerly replicated truncated state into unreplicated keyspace + // in 2.2., but this range may still be using the replicated version + // and we need to make a decision about what to use for the RHS that + // is consistent across the followers: do for the RHS what the LHS + // does: if the LHS has the legacy key, initialize the RHS with a + // legacy key as well. + // + // See VersionUnreplicatedRaftTruncatedState. + truncStateType := stateloader.TruncatedStateUnreplicated + if found, err := engine.MVCCGetProto( + ctx, + batch, + keys.RaftTruncatedStateLegacyKey(rec.GetRangeID()), + hlc.Timestamp{}, + nil, + engine.MVCCGetOptions{}, + ); err != nil { + return enginepb.MVCCStats{}, result.Result{}, errors.Wrap(err, "unable to load legacy truncated state") + } else if found { + truncStateType = stateloader.TruncatedStateLegacyReplicated + } + // Writing the initial state is subtle since this also seeds the Raft // group. It becomes more subtle due to proposer-evaluated Raft. // @@ -944,6 +967,7 @@ func splitTrigger( ctx, batch, rightMS, split.RightDesc, rightLease, *gcThreshold, *txnSpanGCThreshold, rec.ClusterSettings().Version.Version().Version, + truncStateType, ) if err != nil { return enginepb.MVCCStats{}, result.Result{}, errors.Wrap(err, "unable to write initial Replica state") diff --git a/pkg/storage/batcheval/cmd_resolve_intent_test.go b/pkg/storage/batcheval/cmd_resolve_intent_test.go index 505eafe10119..d4d935fb5525 100644 --- a/pkg/storage/batcheval/cmd_resolve_intent_test.go +++ b/pkg/storage/batcheval/cmd_resolve_intent_test.go @@ -36,13 +36,14 @@ import ( ) type mockEvalCtx struct { - clusterSettings *cluster.Settings - desc *roachpb.RangeDescriptor - clock *hlc.Clock - stats enginepb.MVCCStats - qps float64 - abortSpan *abortspan.AbortSpan - gcThreshold hlc.Timestamp + clusterSettings *cluster.Settings + desc *roachpb.RangeDescriptor + clock *hlc.Clock + stats enginepb.MVCCStats + qps float64 + abortSpan *abortspan.AbortSpan + gcThreshold hlc.Timestamp + term, firstIndex uint64 } func (m *mockEvalCtx) String() string { @@ -85,10 +86,10 @@ func (m *mockEvalCtx) IsFirstRange() bool { panic("unimplemented") } func (m *mockEvalCtx) GetFirstIndex() (uint64, error) { - panic("unimplemented") + return m.firstIndex, nil } func (m *mockEvalCtx) GetTerm(uint64) (uint64, error) { - panic("unimplemented") + return m.term, nil } func (m *mockEvalCtx) GetLeaseAppliedIndex() uint64 { panic("unimplemented") diff --git a/pkg/storage/batcheval/cmd_truncate_log.go b/pkg/storage/batcheval/cmd_truncate_log.go index ae61e3d53eb4..1564f849baa4 100644 --- a/pkg/storage/batcheval/cmd_truncate_log.go +++ b/pkg/storage/batcheval/cmd_truncate_log.go @@ -19,10 +19,12 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/batcheval/result" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/storage/spanset" "github.com/cockroachdb/cockroach/pkg/storage/storagepb" + "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/pkg/errors" ) @@ -57,13 +59,33 @@ func TruncateLog( return result.Result{}, nil } - // Have we already truncated this log? If so, just return without an error. - // TODO(tbg): remove this once followers can have divergent truncated states. - firstIndex, err := cArgs.EvalCtx.GetFirstIndex() + var legacyTruncatedState roachpb.RaftTruncatedState + legacyKeyFound, err := engine.MVCCGetProto( + ctx, batch, keys.RaftTruncatedStateLegacyKey(cArgs.EvalCtx.GetRangeID()), + hlc.Timestamp{}, &legacyTruncatedState, engine.MVCCGetOptions{}, + ) if err != nil { return result.Result{}, err } + // See the comment on the cluster version for all the moving parts involved + // in migrating into this cluster version. Note that if the legacy key is + // missing, the cluster version has been bumped (though we may not know it + // yet) and we keep using the unreplicated key. + useNewUnreplicatedTruncatedStateKey := cArgs.EvalCtx.ClusterSettings().Version.IsActive(cluster.VersionUnreplicatedRaftTruncatedState) || !legacyKeyFound + + firstIndex, err := cArgs.EvalCtx.GetFirstIndex() + if err != nil { + return result.Result{}, errors.Wrap(err, "getting first index") + } + // Have we already truncated this log? If so, just return without an error. + // Note that there may in principle be followers whose Raft log is longer + // than this node's, but to issue a truncation we also need the *term* for + // the new truncated state, which we can't obtain if we don't have the log + // entry ourselves. + // + // TODO(tbg): think about synthesizing a valid term. Can we use the next + // existing entry's term? if firstIndex >= args.Index { if log.V(3) { log.Infof(ctx, "attempting to truncate previously truncated raft log. FirstIndex:%d, TruncateFrom:%d", @@ -75,9 +97,16 @@ func TruncateLog( // args.Index is the first index to keep. term, err := cArgs.EvalCtx.GetTerm(args.Index - 1) if err != nil { - return result.Result{}, err + return result.Result{}, errors.Wrap(err, "getting term") } + // Compute the number of bytes freed by this truncation. Note that this will + // only make sense for the leaseholder as we base this off its own first + // index (other replicas may have other first indexes assuming we're not + // still using the legacy truncated state key). In principle, this could be + // off either way, though in practice we don't expect followers to have + // a first index smaller than the leaseholder's (see #34287), and most of + // the time everyone's first index should be the same. start := engine.MakeMVCCMetadataKey(keys.RaftLogKey(rangeID, firstIndex)) end := engine.MakeMVCCMetadataKey(keys.RaftLogKey(rangeID, args.Index)) @@ -110,7 +139,23 @@ func TruncateLog( pd.Replicated.State = &storagepb.ReplicaState{ TruncatedState: tState, } + pd.Replicated.RaftLogDelta = ms.SysBytes - return pd, MakeStateLoader(cArgs.EvalCtx).SetLegacyRaftTruncatedState(ctx, batch, cArgs.Stats, tState) + if !useNewUnreplicatedTruncatedStateKey { + return pd, MakeStateLoader(cArgs.EvalCtx).SetLegacyRaftTruncatedState(ctx, batch, cArgs.Stats, tState) + } + if legacyKeyFound { + // Time to migrate by deleting the legacy key. The downstream-of-Raft + // code will atomically rewrite the truncated state (supplied via the + // side effect) into the new unreplicated key. + if err := engine.MVCCDelete( + ctx, batch, cArgs.Stats, keys.RaftTruncatedStateLegacyKey(cArgs.EvalCtx.GetRangeID()), + hlc.Timestamp{}, nil, /* txn */ + ); err != nil { + return result.Result{}, err + } + } + + return pd, nil } diff --git a/pkg/storage/batcheval/truncate_log_test.go b/pkg/storage/batcheval/truncate_log_test.go new file mode 100644 index 000000000000..e9915697d89d --- /dev/null +++ b/pkg/storage/batcheval/truncate_log_test.go @@ -0,0 +1,200 @@ +// Copyright 2019 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package batcheval + +import ( + "context" + "fmt" + "testing" + + "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/engine" + "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/stretchr/testify/assert" +) + +func putTruncatedState( + t *testing.T, + eng engine.Engine, + rangeID roachpb.RangeID, + truncState roachpb.RaftTruncatedState, + legacy bool, +) { + key := keys.RaftTruncatedStateKey(rangeID) + if legacy { + key = keys.RaftTruncatedStateLegacyKey(rangeID) + } + if err := engine.MVCCPutProto( + context.Background(), eng, nil, key, + hlc.Timestamp{}, nil /* txn */, &truncState, + ); err != nil { + t.Fatal(err) + } +} + +func readTruncStates( + t *testing.T, eng engine.Engine, rangeID roachpb.RangeID, +) (legacy roachpb.RaftTruncatedState, unreplicated roachpb.RaftTruncatedState) { + t.Helper() + legacyFound, err := engine.MVCCGetProto( + context.Background(), eng, keys.RaftTruncatedStateLegacyKey(rangeID), + hlc.Timestamp{}, &legacy, engine.MVCCGetOptions{}, + ) + if err != nil { + t.Fatal(err) + } + if legacyFound != (legacy != roachpb.RaftTruncatedState{}) { + t.Fatalf("legacy key found=%t but state is %+v", legacyFound, legacy) + } + + unreplicatedFound, err := engine.MVCCGetProto( + context.Background(), eng, keys.RaftTruncatedStateKey(rangeID), + hlc.Timestamp{}, &unreplicated, engine.MVCCGetOptions{}, + ) + if err != nil { + t.Fatal(err) + } + if unreplicatedFound != (unreplicated != roachpb.RaftTruncatedState{}) { + t.Fatalf("unreplicated key found=%t but state is %+v", unreplicatedFound, unreplicated) + } + return +} + +const ( + expectationNeither = iota + expectationLegacy + expectationUnreplicated +) + +type unreplicatedTruncStateTest struct { + startsWithLegacy bool + hasVersionBumped bool + exp int // see consts above +} + +func TestTruncateLogUnreplicatedTruncatedState(t *testing.T) { + defer leaktest.AfterTest(t)() + + // Follow the reference below for more information on what's being tested. + _ = cluster.VersionUnreplicatedRaftTruncatedState + + const ( + startsLegacy = true + startsUnreplicated = false + newVersion = true + oldVersion = false + ) + + testCases := []unreplicatedTruncStateTest{ + // Steady states: we have one type of TruncatedState and will end up with + // the same type: either we've already migrated, or we haven't but aren't + // allowed to migrate yet. + {startsUnreplicated, oldVersion, expectationUnreplicated}, + {startsUnreplicated, newVersion, expectationUnreplicated}, + {startsLegacy, oldVersion, expectationLegacy}, + // This is the case in which the migration is triggered. As a result, + // we see neither of the keys written. The new key will be written + // atomically as a side effect (outside of the scope of this test). + {startsLegacy, newVersion, expectationNeither}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%v", tc), func(t *testing.T) { + runUnreplicatedTruncatedState(t, tc) + }) + } +} + +func runUnreplicatedTruncatedState(t *testing.T, tc unreplicatedTruncStateTest) { + ctx := context.Background() + versionOff := cluster.VersionByKey(cluster.VersionUnreplicatedRaftTruncatedState - 1) + versionOn := cluster.VersionByKey(cluster.VersionUnreplicatedRaftTruncatedState) + st := cluster.MakeClusterSettings(versionOff, versionOn) + + if tc.hasVersionBumped { + assert.NoError(t, st.InitializeVersion(cluster.ClusterVersion{Version: versionOn})) + } else { + assert.NoError(t, st.InitializeVersion(cluster.ClusterVersion{Version: versionOff})) + } + + const ( + rangeID = 12 + term = 10 + firstIndex = 100 + ) + + evalCtx := mockEvalCtx{ + clusterSettings: st, + desc: &roachpb.RangeDescriptor{RangeID: rangeID}, + term: term, + firstIndex: firstIndex, + } + + eng := engine.NewInMem(roachpb.Attributes{}, 1<<20) + defer eng.Close() + + truncState := roachpb.RaftTruncatedState{ + Index: firstIndex + 1, + Term: term, + } + + // Put down the TruncatedState specified by the test case. + putTruncatedState(t, eng, rangeID, truncState, tc.startsWithLegacy) + + // Send a truncation request. + req := roachpb.TruncateLogRequest{ + RangeID: rangeID, + Index: firstIndex + 7, + } + cArgs := CommandArgs{ + EvalCtx: &evalCtx, + Args: &req, + } + resp := &roachpb.TruncateLogResponse{} + res, err := TruncateLog(ctx, eng, cArgs, resp) + if err != nil { + t.Fatal(err) + } + + expTruncState := roachpb.RaftTruncatedState{ + Index: req.Index - 1, + Term: term, + } + + legacy, unreplicated := readTruncStates(t, eng, rangeID) + + switch tc.exp { + case expectationLegacy: + assert.Equal(t, expTruncState, legacy) + assert.Zero(t, unreplicated) + case expectationUnreplicated: + // The unreplicated key that we see should be the initial truncated + // state (it's only updated below Raft). + assert.Equal(t, truncState, unreplicated) + assert.Zero(t, legacy) + case expectationNeither: + assert.Zero(t, unreplicated) + assert.Zero(t, legacy) + default: + t.Fatalf("unknown expectation %d", tc.exp) + } + + assert.NotNil(t, res.Replicated.State) + assert.NotNil(t, res.Replicated.State.TruncatedState) + assert.Equal(t, expTruncState, *res.Replicated.State.TruncatedState) +} diff --git a/pkg/storage/below_raft_protos_test.go b/pkg/storage/below_raft_protos_test.go index c2e8f4833d54..69a56db7e817 100644 --- a/pkg/storage/below_raft_protos_test.go +++ b/pkg/storage/below_raft_protos_test.go @@ -118,6 +118,17 @@ var belowRaftGoldenProtos = map[reflect.Type]fixture{ emptySum: 892800390935990883, populatedSum: 16231745342114354146, }, + // This is used downstream of Raft only to write it into unreplicated keyspace + // as part of VersionUnreplicatedRaftTruncatedState. + // However, it has been sent through Raft for a long time, as part of + // ReplicatedEvalResult. + reflect.TypeOf(&roachpb.RaftTruncatedState{}): { + populatedConstructor: func(r *rand.Rand) protoutil.Message { + return roachpb.NewPopulatedRaftTruncatedState(r, false) + }, + emptySum: 5531676819244041709, + populatedSum: 14781226418259198098, + }, } func TestBelowRaftProtos(t *testing.T) { diff --git a/pkg/storage/engine/engine.go b/pkg/storage/engine/engine.go index 00932fe2668e..b0ffa5e33bcb 100644 --- a/pkg/storage/engine/engine.go +++ b/pkg/storage/engine/engine.go @@ -352,11 +352,19 @@ type Batch interface { // Distinct returns a view of the existing batch which only sees writes that // were performed before the Distinct batch was created. That is, the // returned batch will not read its own writes, but it will read writes to - // the parent batch performed before the call to Distinct(). The returned + // the parent batch performed before the call to Distinct(), except if the + // parent batch is a WriteOnlyBatch, in which case the Distinct() batch will + // read from the underlying engine. + // + // The returned // batch needs to be closed before using the parent batch again. This is used // as an optimization to avoid flushing mutations buffered by the batch in // situations where we know all of the batched operations are for distinct // keys. + // + // TODO(tbg): it seems insane that you cannot read from a WriteOnlyBatch but + // you can read from a Distinct on top of a WriteOnlyBatch but randomly don't + // see the batch at all. I was personally just bitten by this. Distinct() ReadWriter // Empty returns whether the batch has been written to or not. Empty() bool diff --git a/pkg/storage/main_test.go b/pkg/storage/main_test.go index f333f5f48d67..551d87a8616b 100644 --- a/pkg/storage/main_test.go +++ b/pkg/storage/main_test.go @@ -68,7 +68,7 @@ func TestMain(m *testing.M) { delete(notBelowRaftProtos, typ) } else { failed = true - fmt.Printf("%s: missing fixture!\n", typ) + fmt.Printf("%s: missing fixture! Please adjust belowRaftGoldenProtos if necessary\n", typ) } } diff --git a/pkg/storage/raft.pb.go b/pkg/storage/raft.pb.go index d231257348f1..80b533dc71f9 100644 --- a/pkg/storage/raft.pb.go +++ b/pkg/storage/raft.pb.go @@ -70,7 +70,7 @@ func (x *SnapshotRequest_Priority) UnmarshalJSON(data []byte) error { return nil } func (SnapshotRequest_Priority) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{5, 0} + return fileDescriptor_raft_06448cf81da2fcd5, []int{5, 0} } type SnapshotRequest_Strategy int32 @@ -107,7 +107,7 @@ func (x *SnapshotRequest_Strategy) UnmarshalJSON(data []byte) error { return nil } func (SnapshotRequest_Strategy) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{5, 1} + return fileDescriptor_raft_06448cf81da2fcd5, []int{5, 1} } type SnapshotResponse_Status int32 @@ -152,7 +152,7 @@ func (x *SnapshotResponse_Status) UnmarshalJSON(data []byte) error { return nil } func (SnapshotResponse_Status) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{6, 0} + return fileDescriptor_raft_06448cf81da2fcd5, []int{6, 0} } // RaftHeartbeat is a request that contains the barebones information for a @@ -174,7 +174,7 @@ func (m *RaftHeartbeat) Reset() { *m = RaftHeartbeat{} } func (m *RaftHeartbeat) String() string { return proto.CompactTextString(m) } func (*RaftHeartbeat) ProtoMessage() {} func (*RaftHeartbeat) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{0} + return fileDescriptor_raft_06448cf81da2fcd5, []int{0} } func (m *RaftHeartbeat) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -232,7 +232,7 @@ func (m *RaftMessageRequest) Reset() { *m = RaftMessageRequest{} } func (m *RaftMessageRequest) String() string { return proto.CompactTextString(m) } func (*RaftMessageRequest) ProtoMessage() {} func (*RaftMessageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{1} + return fileDescriptor_raft_06448cf81da2fcd5, []int{1} } func (m *RaftMessageRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -267,7 +267,7 @@ func (m *RaftMessageRequestBatch) Reset() { *m = RaftMessageRequestBatch func (m *RaftMessageRequestBatch) String() string { return proto.CompactTextString(m) } func (*RaftMessageRequestBatch) ProtoMessage() {} func (*RaftMessageRequestBatch) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{2} + return fileDescriptor_raft_06448cf81da2fcd5, []int{2} } func (m *RaftMessageRequestBatch) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -302,7 +302,7 @@ func (m *RaftMessageResponseUnion) Reset() { *m = RaftMessageResponseUni func (m *RaftMessageResponseUnion) String() string { return proto.CompactTextString(m) } func (*RaftMessageResponseUnion) ProtoMessage() {} func (*RaftMessageResponseUnion) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{3} + return fileDescriptor_raft_06448cf81da2fcd5, []int{3} } func (m *RaftMessageResponseUnion) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -347,7 +347,7 @@ func (m *RaftMessageResponse) Reset() { *m = RaftMessageResponse{} } func (m *RaftMessageResponse) String() string { return proto.CompactTextString(m) } func (*RaftMessageResponse) ProtoMessage() {} func (*RaftMessageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{4} + return fileDescriptor_raft_06448cf81da2fcd5, []int{4} } func (m *RaftMessageResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -390,7 +390,7 @@ func (m *SnapshotRequest) Reset() { *m = SnapshotRequest{} } func (m *SnapshotRequest) String() string { return proto.CompactTextString(m) } func (*SnapshotRequest) ProtoMessage() {} func (*SnapshotRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{5} + return fileDescriptor_raft_06448cf81da2fcd5, []int{5} } func (m *SnapshotRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -433,16 +433,26 @@ type SnapshotRequest_Header struct { // The priority of the snapshot. Priority SnapshotRequest_Priority `protobuf:"varint,6,opt,name=priority,enum=cockroach.storage.SnapshotRequest_Priority" json:"priority"` // The strategy of the snapshot. - Strategy SnapshotRequest_Strategy `protobuf:"varint,7,opt,name=strategy,enum=cockroach.storage.SnapshotRequest_Strategy" json:"strategy"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + Strategy SnapshotRequest_Strategy `protobuf:"varint,7,opt,name=strategy,enum=cockroach.storage.SnapshotRequest_Strategy" json:"strategy"` + // Whether the snapshot uses the unreplicated RaftTruncatedState or not. + // This is generally always true at 2.2 and above outside of the migration + // phase, though theoretically it could take a long time for all ranges + // to update to the new mechanism. This bool is true iff the Raft log at + // the snapshot's applied index is using the new key. In particular, it + // is true if the index itself carries out the migration (in which case + // the data in the snapshot contains neither key). + // + // See VersionUnreplicatedRaftTruncatedState. + UnreplicatedTruncatedState bool `protobuf:"varint,8,opt,name=unreplicated_truncated_state,json=unreplicatedTruncatedState" json:"unreplicated_truncated_state"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *SnapshotRequest_Header) Reset() { *m = SnapshotRequest_Header{} } func (m *SnapshotRequest_Header) String() string { return proto.CompactTextString(m) } func (*SnapshotRequest_Header) ProtoMessage() {} func (*SnapshotRequest_Header) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{5, 0} + return fileDescriptor_raft_06448cf81da2fcd5, []int{5, 0} } func (m *SnapshotRequest_Header) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -478,7 +488,7 @@ func (m *SnapshotResponse) Reset() { *m = SnapshotResponse{} } func (m *SnapshotResponse) String() string { return proto.CompactTextString(m) } func (*SnapshotResponse) ProtoMessage() {} func (*SnapshotResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{6} + return fileDescriptor_raft_06448cf81da2fcd5, []int{6} } func (m *SnapshotResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -519,7 +529,7 @@ func (m *ConfChangeContext) Reset() { *m = ConfChangeContext{} } func (m *ConfChangeContext) String() string { return proto.CompactTextString(m) } func (*ConfChangeContext) ProtoMessage() {} func (*ConfChangeContext) Descriptor() ([]byte, []int) { - return fileDescriptor_raft_460a63b017d715a3, []int{7} + return fileDescriptor_raft_06448cf81da2fcd5, []int{7} } func (m *ConfChangeContext) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1053,6 +1063,14 @@ func (m *SnapshotRequest_Header) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x38 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Strategy)) + dAtA[i] = 0x40 + i++ + if m.UnreplicatedTruncatedState { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ return i, nil } @@ -1256,6 +1274,7 @@ func (m *SnapshotRequest_Header) Size() (n int) { n += 1 + l + sovRaft(uint64(l)) n += 1 + sovRaft(uint64(m.Priority)) n += 1 + sovRaft(uint64(m.Strategy)) + n += 2 return n } @@ -2406,6 +2425,26 @@ func (m *SnapshotRequest_Header) Unmarshal(dAtA []byte) error { break } } + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UnreplicatedTruncatedState", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.UnreplicatedTruncatedState = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRaft(dAtA[iNdEx:]) @@ -2770,80 +2809,82 @@ var ( ErrIntOverflowRaft = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("storage/raft.proto", fileDescriptor_raft_460a63b017d715a3) } - -var fileDescriptor_raft_460a63b017d715a3 = []byte{ - // 1147 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x56, 0xdd, 0x6e, 0x1b, 0xc5, - 0x17, 0xf7, 0xc6, 0xdf, 0xc7, 0x76, 0xb3, 0x9d, 0x7f, 0xf5, 0x67, 0x65, 0xc0, 0x36, 0x5b, 0x5a, - 0x99, 0x22, 0xad, 0x8b, 0x55, 0xb8, 0xe0, 0xce, 0x1f, 0xdb, 0xc6, 0x4d, 0xf3, 0xa1, 0x4d, 0x5b, - 0x04, 0x52, 0x65, 0x8d, 0xd7, 0x63, 0x7b, 0x15, 0x7b, 0x67, 0xbb, 0x3b, 0x2e, 0xb8, 0x4f, 0xc1, - 0x13, 0x20, 0x6e, 0xb8, 0xe1, 0x05, 0x78, 0x85, 0xdc, 0x20, 0x71, 0x59, 0x09, 0x14, 0x41, 0x78, - 0x8b, 0x5e, 0xa1, 0x99, 0x9d, 0x71, 0x9c, 0xc4, 0xd0, 0x04, 0x21, 0x6e, 0xb8, 0x49, 0xbc, 0xe7, - 0xcc, 0xef, 0x77, 0xf6, 0x9c, 0xdf, 0x39, 0x67, 0x16, 0x50, 0xc4, 0x68, 0x88, 0xc7, 0xa4, 0x11, - 0xe2, 0x11, 0xb3, 0x82, 0x90, 0x32, 0x8a, 0xae, 0xbb, 0xd4, 0x3d, 0x0c, 0x29, 0x76, 0x27, 0x96, - 0xf4, 0x96, 0x6f, 0x88, 0xc7, 0x60, 0xd0, 0x20, 0x61, 0x48, 0xc3, 0x28, 0x3e, 0x58, 0xfe, 0xbf, - 0xb2, 0xce, 0x08, 0xc3, 0x43, 0xcc, 0xb0, 0xb4, 0xbf, 0xab, 0x48, 0xe5, 0xff, 0x60, 0xd0, 0x88, - 0x18, 0x66, 0x44, 0xba, 0xdf, 0x26, 0xcc, 0x1d, 0x8a, 0x80, 0xe2, 0x4f, 0x30, 0x58, 0x09, 0x5e, - 0xbe, 0x31, 0xa6, 0x63, 0x2a, 0x7e, 0x36, 0xf8, 0xaf, 0xd8, 0x6a, 0xfe, 0x90, 0x84, 0x92, 0x83, - 0x47, 0x6c, 0x8b, 0xe0, 0x90, 0x0d, 0x08, 0x66, 0x68, 0x00, 0xb9, 0x10, 0xfb, 0x63, 0xd2, 0xf7, - 0x86, 0x86, 0x56, 0xd3, 0xea, 0xa9, 0xf6, 0x83, 0xa3, 0xe3, 0x6a, 0xe2, 0xe4, 0xb8, 0x9a, 0x75, - 0xb8, 0xbd, 0xd7, 0x7d, 0x7d, 0x5c, 0xbd, 0x37, 0xf6, 0xd8, 0x64, 0x3e, 0xb0, 0x5c, 0x3a, 0x6b, - 0x2c, 0x93, 0x1a, 0x0e, 0x4e, 0x7f, 0x37, 0x82, 0xc3, 0x71, 0x43, 0x66, 0x61, 0x49, 0x9c, 0x93, - 0x15, 0xc4, 0xbd, 0x21, 0xfa, 0x12, 0x36, 0x47, 0x21, 0x9d, 0xf5, 0x43, 0x12, 0x4c, 0x3d, 0x17, - 0xf3, 0x50, 0x1b, 0x35, 0xad, 0x5e, 0x6a, 0xef, 0xc9, 0x50, 0xa5, 0xfb, 0x21, 0x9d, 0x39, 0xb1, - 0x57, 0x04, 0xfc, 0xe4, 0x6a, 0x01, 0x15, 0xd2, 0x29, 0x8d, 0x56, 0x88, 0x86, 0xe8, 0x39, 0x94, - 0x18, 0x5d, 0x0d, 0x9b, 0x14, 0x61, 0x77, 0x64, 0xd8, 0xc2, 0x63, 0xfa, 0x4f, 0x04, 0x2d, 0x30, - 0x7a, 0x1a, 0xd2, 0x80, 0x14, 0x23, 0xe1, 0xcc, 0x48, 0x89, 0x5a, 0xa6, 0x78, 0x24, 0x47, 0x58, - 0xd0, 0x3b, 0x90, 0x71, 0xe9, 0x6c, 0xe6, 0x31, 0x23, 0xbd, 0xe2, 0x93, 0x36, 0x54, 0x81, 0xec, - 0xf3, 0xb9, 0x47, 0x22, 0x97, 0x18, 0x99, 0x9a, 0x56, 0xcf, 0x49, 0xb7, 0x32, 0x9a, 0x3f, 0xa7, - 0x00, 0x71, 0xe5, 0x76, 0x48, 0x14, 0xe1, 0x31, 0x71, 0xc8, 0xf3, 0x39, 0x89, 0xfe, 0x1d, 0xf9, - 0x76, 0xa0, 0xb8, 0x2a, 0x9f, 0xd0, 0xae, 0xd0, 0x7c, 0xdf, 0x3a, 0x6d, 0xef, 0x73, 0x35, 0xe9, - 0x92, 0xc8, 0x0d, 0xbd, 0x80, 0xd1, 0x50, 0x66, 0x51, 0x58, 0x91, 0x05, 0xf5, 0x00, 0x4e, 0x45, - 0x11, 0x8a, 0x5c, 0x8d, 0x2c, 0xbf, 0x2c, 0x37, 0x6a, 0x40, 0x76, 0x16, 0xd7, 0x43, 0xd4, 0xbb, - 0xd0, 0xdc, 0xb4, 0xe2, 0x49, 0xb0, 0x64, 0x99, 0x54, 0x15, 0xe5, 0xa9, 0xd5, 0x2a, 0xa7, 0xd7, - 0x54, 0x19, 0xdd, 0x07, 0x98, 0xa8, 0xd1, 0x88, 0x8c, 0x4c, 0x2d, 0x59, 0x2f, 0x34, 0x6b, 0xd6, - 0x85, 0x39, 0xb6, 0xce, 0xcc, 0x90, 0x24, 0x59, 0x41, 0xa2, 0x3d, 0xd8, 0x5c, 0x3e, 0xf5, 0x43, - 0x12, 0x05, 0x91, 0x91, 0xbd, 0x12, 0xd9, 0xb5, 0x25, 0xdc, 0xe1, 0x68, 0xf4, 0x0c, 0x36, 0x63, - 0x9d, 0x23, 0x86, 0x43, 0xd6, 0x3f, 0x24, 0x0b, 0x23, 0x57, 0xd3, 0xea, 0xc5, 0xf6, 0xc7, 0xaf, - 0x8f, 0xab, 0x1f, 0x5d, 0x4d, 0xdf, 0x6d, 0xb2, 0x70, 0x4a, 0x82, 0xed, 0x80, 0x93, 0x6d, 0x93, - 0x85, 0x39, 0x80, 0xb7, 0x2e, 0x36, 0x57, 0x1b, 0x33, 0x77, 0x82, 0x1e, 0x40, 0x2e, 0x8c, 0x9f, - 0x23, 0x43, 0x13, 0x39, 0xdc, 0xfa, 0x93, 0x1c, 0xce, 0xa1, 0xe3, 0x44, 0x96, 0x60, 0x73, 0x1f, - 0x8c, 0x33, 0xa7, 0xa2, 0x80, 0xfa, 0x11, 0x79, 0xe2, 0x7b, 0xd4, 0x47, 0x16, 0xa4, 0xc5, 0x46, - 0x14, 0x3d, 0x5c, 0x68, 0x1a, 0x6b, 0xda, 0xc1, 0xe6, 0x7e, 0x27, 0x3e, 0xf6, 0x69, 0xea, 0xe8, - 0xdb, 0xaa, 0x66, 0xfe, 0xb2, 0x01, 0xff, 0x5b, 0x43, 0xf9, 0x1f, 0x1f, 0x8a, 0x07, 0x90, 0x9e, - 0xf3, 0xa2, 0xca, 0x91, 0xf8, 0xf0, 0x4d, 0x6a, 0xad, 0xe8, 0x20, 0xc9, 0x62, 0xbc, 0xf9, 0x7d, - 0x1a, 0x36, 0x0f, 0x7c, 0x1c, 0x44, 0x13, 0xca, 0xd4, 0xbe, 0x69, 0x41, 0x66, 0x42, 0xf0, 0x90, - 0x28, 0xa5, 0x3e, 0x58, 0xc3, 0x7e, 0x0e, 0x63, 0x6d, 0x09, 0x80, 0x23, 0x81, 0xe8, 0x36, 0xe4, - 0x0e, 0x5f, 0xf4, 0x07, 0xbc, 0xb9, 0x44, 0xd5, 0x8a, 0xed, 0x02, 0x57, 0x66, 0xfb, 0xa9, 0xe8, - 0x37, 0x27, 0x7b, 0xf8, 0x22, 0x6e, 0xbc, 0x2a, 0x14, 0xa6, 0x74, 0xdc, 0x27, 0x3e, 0x0b, 0x3d, - 0x12, 0x19, 0xc9, 0x5a, 0xb2, 0x5e, 0x74, 0x60, 0x4a, 0xc7, 0x76, 0x6c, 0x41, 0x65, 0x48, 0x8f, - 0x3c, 0x1f, 0x4f, 0x45, 0xa2, 0x6a, 0x94, 0x63, 0x53, 0xf9, 0x9b, 0x24, 0x64, 0xe2, 0xb8, 0xe8, - 0x19, 0xdc, 0xe0, 0x4b, 0xa1, 0x2f, 0x77, 0x40, 0x5f, 0x36, 0xa4, 0x54, 0xec, 0x4a, 0xcd, 0x8c, - 0xc2, 0x8b, 0x1b, 0xf8, 0x26, 0x80, 0x9c, 0x4c, 0xef, 0x25, 0x11, 0xca, 0x25, 0x95, 0x26, 0xf1, - 0x8c, 0x79, 0x2f, 0x09, 0xba, 0x05, 0x05, 0x17, 0xfb, 0xfd, 0x21, 0x71, 0xa7, 0x9e, 0x4f, 0xce, - 0xbc, 0x30, 0xb8, 0xd8, 0xef, 0xc6, 0x76, 0x64, 0x43, 0x5a, 0x5c, 0xf0, 0x62, 0x39, 0xad, 0x2f, - 0xee, 0xf2, 0x53, 0x40, 0xb5, 0xc2, 0x01, 0x07, 0xa8, 0xe4, 0x05, 0x1a, 0xed, 0x40, 0x2e, 0x08, - 0x3d, 0x1a, 0x7a, 0x6c, 0x21, 0x2e, 0x93, 0x6b, 0x6b, 0x9b, 0xe0, 0xbc, 0x4c, 0xfb, 0x12, 0xa2, - 0x06, 0x57, 0x51, 0x70, 0xba, 0x88, 0x85, 0x98, 0x91, 0xf1, 0xc2, 0xc8, 0x5e, 0x9a, 0xee, 0x40, - 0x42, 0x14, 0x9d, 0xa2, 0x78, 0x98, 0xca, 0x69, 0xfa, 0x86, 0x79, 0x0f, 0x72, 0x2a, 0x20, 0x2a, - 0x40, 0xf6, 0xc9, 0xee, 0xf6, 0xee, 0xde, 0x67, 0xbb, 0x7a, 0x02, 0x15, 0x21, 0xe7, 0xd8, 0x9d, - 0xbd, 0xa7, 0xb6, 0xf3, 0xb9, 0xae, 0xa1, 0x12, 0xe4, 0x1d, 0xbb, 0xdd, 0x7a, 0xd4, 0xda, 0xed, - 0xd8, 0xfa, 0x86, 0x69, 0x40, 0x4e, 0xf1, 0xf2, 0x83, 0xdb, 0x4f, 0xfb, 0xed, 0xd6, 0xe3, 0xce, - 0x96, 0x9e, 0x30, 0x7f, 0xd4, 0x40, 0x3f, 0x7d, 0x05, 0xb9, 0x08, 0xb6, 0x20, 0xc3, 0x2b, 0x32, - 0x8f, 0x44, 0xb7, 0x5e, 0x6b, 0xde, 0xf9, 0xcb, 0xf7, 0x8e, 0x41, 0xd6, 0x81, 0x40, 0xa8, 0xeb, - 0x39, 0xc6, 0xf3, 0x8b, 0x43, 0xdd, 0x34, 0xbc, 0x6f, 0xf2, 0xe7, 0x2e, 0x16, 0xb3, 0x07, 0x99, - 0x18, 0x77, 0x21, 0x99, 0x56, 0xa7, 0x63, 0xef, 0x3f, 0xb6, 0xbb, 0xba, 0xc6, 0x5d, 0xad, 0xfd, - 0xfd, 0x47, 0x3d, 0xbb, 0xab, 0x6f, 0xa0, 0x3c, 0xa4, 0x6d, 0xc7, 0xd9, 0x73, 0xf4, 0x24, 0x3f, - 0xd5, 0xb5, 0x3b, 0x8f, 0x7a, 0xbb, 0x76, 0x57, 0x4f, 0x3d, 0x4c, 0xe5, 0x92, 0x7a, 0xca, 0xfc, - 0x4e, 0x83, 0xeb, 0x1d, 0xea, 0x8f, 0x3a, 0x13, 0xde, 0x44, 0x1d, 0xea, 0x33, 0xf2, 0x15, 0x43, - 0x77, 0x01, 0xf8, 0xf7, 0x02, 0xf6, 0x87, 0x6a, 0xb7, 0xe5, 0xdb, 0xd7, 0xe5, 0x6e, 0xcb, 0x77, - 0x62, 0x4f, 0xaf, 0xeb, 0xe4, 0xe5, 0x21, 0xf1, 0x3d, 0x92, 0x0d, 0xf0, 0x62, 0x4a, 0x71, 0xfc, - 0xcd, 0x55, 0x74, 0xd4, 0x23, 0xea, 0x42, 0xf6, 0xef, 0xef, 0x1b, 0x05, 0x6d, 0xbe, 0xd2, 0x20, - 0xbf, 0x33, 0x9f, 0x32, 0x8f, 0x0f, 0x0d, 0x9a, 0x82, 0xbe, 0x32, 0x3c, 0xf1, 0x1c, 0xdf, 0xb9, - 0xdc, 0x84, 0xf1, 0xb3, 0xe5, 0xdb, 0x97, 0x5b, 0x56, 0x66, 0xa2, 0xae, 0xdd, 0xd5, 0xd0, 0x33, - 0x28, 0x72, 0xa7, 0x52, 0x10, 0x99, 0x6f, 0x6e, 0xcb, 0xf2, 0xcd, 0x4b, 0xb4, 0x40, 0x4c, 0xdf, - 0x7e, 0xef, 0xe8, 0xb7, 0x4a, 0xe2, 0xe8, 0xa4, 0xa2, 0xfd, 0x74, 0x52, 0xd1, 0x5e, 0x9d, 0x54, - 0xb4, 0x5f, 0x4f, 0x2a, 0xda, 0xd7, 0xbf, 0x57, 0x12, 0x5f, 0x64, 0x25, 0xf2, 0x8f, 0x00, 0x00, - 0x00, 0xff, 0xff, 0xd4, 0x72, 0xff, 0x08, 0xf7, 0x0b, 0x00, 0x00, +func init() { proto.RegisterFile("storage/raft.proto", fileDescriptor_raft_06448cf81da2fcd5) } + +var fileDescriptor_raft_06448cf81da2fcd5 = []byte{ + // 1172 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x56, 0xdd, 0x6e, 0x1b, 0x55, + 0x10, 0xce, 0xc6, 0xff, 0x63, 0xbb, 0x71, 0x0f, 0x15, 0xac, 0x4c, 0x71, 0xcc, 0x96, 0x56, 0xa6, + 0x48, 0x76, 0x89, 0x0a, 0x17, 0xdc, 0xc5, 0xf6, 0xb6, 0x71, 0xd3, 0xfc, 0x68, 0x93, 0x16, 0x81, + 0x54, 0x59, 0xc7, 0xeb, 0x13, 0x7b, 0x15, 0x7b, 0xcf, 0xf6, 0xec, 0x71, 0x21, 0x7d, 0x0a, 0x1e, + 0x81, 0x1b, 0x9e, 0x81, 0x57, 0xc8, 0x0d, 0x12, 0x97, 0x95, 0x40, 0x11, 0x04, 0x89, 0x07, 0xe0, + 0xb2, 0x57, 0xe8, 0xfc, 0x25, 0x9b, 0xc4, 0xd0, 0x04, 0x21, 0x6e, 0xb8, 0xb1, 0x77, 0x67, 0xe6, + 0x9b, 0xd9, 0x99, 0x6f, 0x66, 0xce, 0x01, 0x14, 0x73, 0xca, 0xf0, 0x88, 0xb4, 0x18, 0xde, 0xe3, + 0xcd, 0x88, 0x51, 0x4e, 0xd1, 0x75, 0x9f, 0xfa, 0xfb, 0x8c, 0x62, 0x7f, 0xdc, 0xd4, 0xda, 0xea, + 0x0d, 0xf9, 0x1a, 0x0d, 0x5a, 0x84, 0x31, 0xca, 0x62, 0x65, 0x58, 0x7d, 0xdb, 0x48, 0xa7, 0x84, + 0xe3, 0x21, 0xe6, 0x58, 0xcb, 0xdf, 0x33, 0x4e, 0xf5, 0x7f, 0x34, 0x68, 0xc5, 0x1c, 0x73, 0xa2, + 0xd5, 0xef, 0x12, 0xee, 0x0f, 0x65, 0x40, 0xf9, 0x13, 0x0d, 0x12, 0xc1, 0xab, 0x37, 0x46, 0x74, + 0x44, 0xe5, 0x63, 0x4b, 0x3c, 0x29, 0xa9, 0xf3, 0x7d, 0x0a, 0xca, 0x1e, 0xde, 0xe3, 0x6b, 0x04, + 0x33, 0x3e, 0x20, 0x98, 0xa3, 0x01, 0xe4, 0x19, 0x0e, 0x47, 0xa4, 0x1f, 0x0c, 0x6d, 0xab, 0x6e, + 0x35, 0xd2, 0xed, 0x87, 0x87, 0x47, 0xcb, 0x0b, 0xc7, 0x47, 0xcb, 0x39, 0x4f, 0xc8, 0x7b, 0xdd, + 0xd7, 0x47, 0xcb, 0xf7, 0x47, 0x01, 0x1f, 0xcf, 0x06, 0x4d, 0x9f, 0x4e, 0x5b, 0x27, 0x49, 0x0d, + 0x07, 0xa7, 0xcf, 0xad, 0x68, 0x7f, 0xd4, 0xd2, 0x59, 0x34, 0x35, 0xce, 0xcb, 0x49, 0xc7, 0xbd, + 0x21, 0xfa, 0x0a, 0x96, 0xf6, 0x18, 0x9d, 0xf6, 0x19, 0x89, 0x26, 0x81, 0x8f, 0x45, 0xa8, 0xc5, + 0xba, 0xd5, 0x28, 0xb7, 0xb7, 0x74, 0xa8, 0xf2, 0x03, 0x46, 0xa7, 0x9e, 0xd2, 0xca, 0x80, 0x9f, + 0x5e, 0x2d, 0xa0, 0x41, 0x7a, 0xe5, 0xbd, 0x84, 0xa3, 0x21, 0x7a, 0x0e, 0x65, 0x4e, 0x93, 0x61, + 0x53, 0x32, 0xec, 0x86, 0x0e, 0x5b, 0xdc, 0xa5, 0xff, 0x46, 0xd0, 0x22, 0xa7, 0xa7, 0x21, 0x6d, + 0x48, 0x73, 0xc2, 0xa6, 0x76, 0x5a, 0xd6, 0x32, 0x2d, 0x22, 0x79, 0x52, 0x82, 0x6e, 0x42, 0xd6, + 0xa7, 0xd3, 0x69, 0xc0, 0xed, 0x4c, 0x42, 0xa7, 0x65, 0xa8, 0x06, 0xb9, 0xe7, 0xb3, 0x80, 0xc4, + 0x3e, 0xb1, 0xb3, 0x75, 0xab, 0x91, 0xd7, 0x6a, 0x23, 0x74, 0x7e, 0x4a, 0x03, 0x12, 0xcc, 0x6d, + 0x90, 0x38, 0xc6, 0x23, 0xe2, 0x91, 0xe7, 0x33, 0x12, 0xff, 0x37, 0xf4, 0x6d, 0x40, 0x29, 0x49, + 0x9f, 0xe4, 0xae, 0xb8, 0xf2, 0x41, 0xf3, 0xb4, 0xbd, 0xcf, 0xd5, 0xa4, 0x4b, 0x62, 0x9f, 0x05, + 0x11, 0xa7, 0x4c, 0x67, 0x51, 0x4c, 0xd0, 0x82, 0x7a, 0x00, 0xa7, 0xa4, 0x48, 0x46, 0xae, 0xe6, + 0xac, 0x70, 0x52, 0x6e, 0xd4, 0x82, 0xdc, 0x54, 0xd5, 0x43, 0xd6, 0xbb, 0xb8, 0xb2, 0xd4, 0x54, + 0x93, 0xd0, 0xd4, 0x65, 0x32, 0x55, 0xd4, 0x56, 0xc9, 0x2a, 0x67, 0xe6, 0x54, 0x19, 0x3d, 0x00, + 0x18, 0x9b, 0xd1, 0x88, 0xed, 0x6c, 0x3d, 0xd5, 0x28, 0xae, 0xd4, 0x9b, 0x17, 0xe6, 0xb8, 0x79, + 0x66, 0x86, 0xb4, 0x93, 0x04, 0x12, 0x6d, 0xc1, 0xd2, 0xc9, 0x5b, 0x9f, 0x91, 0x38, 0x8a, 0xed, + 0xdc, 0x95, 0x9c, 0x5d, 0x3b, 0x81, 0x7b, 0x02, 0x8d, 0x9e, 0xc1, 0x92, 0xe2, 0x39, 0xe6, 0x98, + 0xf1, 0xfe, 0x3e, 0x39, 0xb0, 0xf3, 0x75, 0xab, 0x51, 0x6a, 0x7f, 0xf2, 0xfa, 0x68, 0xf9, 0xe3, + 0xab, 0xf1, 0xbb, 0x4e, 0x0e, 0xbc, 0xb2, 0xf4, 0xb6, 0x23, 0x9c, 0xad, 0x93, 0x03, 0x67, 0x00, + 0xef, 0x5c, 0x6c, 0xae, 0x36, 0xe6, 0xfe, 0x18, 0x3d, 0x84, 0x3c, 0x53, 0xef, 0xb1, 0x6d, 0xc9, + 0x1c, 0x6e, 0xff, 0x45, 0x0e, 0xe7, 0xd0, 0x2a, 0x91, 0x13, 0xb0, 0xb3, 0x0d, 0xf6, 0x19, 0xab, + 0x38, 0xa2, 0x61, 0x4c, 0x9e, 0x84, 0x01, 0x0d, 0x51, 0x13, 0x32, 0x72, 0x23, 0xca, 0x1e, 0x2e, + 0xae, 0xd8, 0x73, 0xda, 0xc1, 0x15, 0x7a, 0x4f, 0x99, 0x7d, 0x96, 0x3e, 0xfc, 0x76, 0xd9, 0x72, + 0x7e, 0x5e, 0x84, 0xb7, 0xe6, 0xb8, 0xfc, 0x9f, 0x0f, 0xc5, 0x43, 0xc8, 0xcc, 0x44, 0x51, 0xf5, + 0x48, 0x7c, 0xf4, 0x26, 0xb6, 0x12, 0x3c, 0x68, 0x67, 0x0a, 0xef, 0xfc, 0x91, 0x81, 0xa5, 0x9d, + 0x10, 0x47, 0xf1, 0x98, 0x72, 0xb3, 0x6f, 0x56, 0x21, 0x3b, 0x26, 0x78, 0x48, 0x0c, 0x53, 0x1f, + 0xce, 0xf1, 0x7e, 0x0e, 0xd3, 0x5c, 0x93, 0x00, 0x4f, 0x03, 0xd1, 0x1d, 0xc8, 0xef, 0xbf, 0xe8, + 0x0f, 0x44, 0x73, 0xc9, 0xaa, 0x95, 0xda, 0x45, 0xc1, 0xcc, 0xfa, 0x53, 0xd9, 0x6f, 0x5e, 0x6e, + 0xff, 0x85, 0x6a, 0xbc, 0x65, 0x28, 0x4e, 0xe8, 0xa8, 0x4f, 0x42, 0xce, 0x02, 0x12, 0xdb, 0xa9, + 0x7a, 0xaa, 0x51, 0xf2, 0x60, 0x42, 0x47, 0xae, 0x92, 0xa0, 0x2a, 0x64, 0xf6, 0x82, 0x10, 0x4f, + 0x64, 0xa2, 0x66, 0x94, 0x95, 0xa8, 0xfa, 0x7b, 0x0a, 0xb2, 0x2a, 0x2e, 0x7a, 0x06, 0x37, 0xc4, + 0x52, 0xe8, 0xeb, 0x1d, 0xd0, 0xd7, 0x0d, 0xa9, 0x19, 0xbb, 0x52, 0x33, 0x23, 0x76, 0x71, 0x03, + 0xdf, 0x02, 0xd0, 0x93, 0x19, 0xbc, 0x24, 0x92, 0xb9, 0x94, 0xe1, 0x44, 0xcd, 0x58, 0xf0, 0x92, + 0xa0, 0xdb, 0x50, 0xf4, 0x71, 0xd8, 0x1f, 0x12, 0x7f, 0x12, 0x84, 0xe4, 0xcc, 0x07, 0x83, 0x8f, + 0xc3, 0xae, 0x92, 0x23, 0x17, 0x32, 0xf2, 0x80, 0x97, 0xcb, 0x69, 0x7e, 0x71, 0x4f, 0xae, 0x02, + 0xa6, 0x15, 0x76, 0x04, 0xc0, 0x24, 0x2f, 0xd1, 0x68, 0x03, 0xf2, 0x11, 0x0b, 0x28, 0x0b, 0xf8, + 0x81, 0x3c, 0x4c, 0xae, 0xcd, 0x6d, 0x82, 0xf3, 0x34, 0x6d, 0x6b, 0x88, 0x19, 0x5c, 0xe3, 0x42, + 0xb8, 0x8b, 0x39, 0xc3, 0x9c, 0x8c, 0x0e, 0xec, 0xdc, 0xa5, 0xdd, 0xed, 0x68, 0x88, 0x71, 0x67, + 0x5c, 0xa0, 0x07, 0x70, 0x73, 0x16, 0xea, 0x4e, 0xe7, 0x64, 0xd8, 0xe7, 0x6c, 0x16, 0xaa, 0x27, + 0x95, 0x7b, 0x3e, 0x51, 0x9c, 0x6a, 0xd2, 0x72, 0xd7, 0x18, 0xca, 0x94, 0x1f, 0xa5, 0xf3, 0x56, + 0x65, 0xd1, 0xb9, 0x0f, 0x79, 0xf3, 0xe1, 0xa8, 0x08, 0xb9, 0x27, 0x9b, 0xeb, 0x9b, 0x5b, 0x9f, + 0x6f, 0x56, 0x16, 0x50, 0x09, 0xf2, 0x9e, 0xdb, 0xd9, 0x7a, 0xea, 0x7a, 0x5f, 0x54, 0x2c, 0x54, + 0x86, 0x82, 0xe7, 0xb6, 0x57, 0x1f, 0xaf, 0x6e, 0x76, 0xdc, 0xca, 0xa2, 0x63, 0x43, 0xde, 0x7c, + 0x9f, 0x30, 0x5c, 0x7f, 0xda, 0x6f, 0xaf, 0xee, 0x76, 0xd6, 0x2a, 0x0b, 0xce, 0x0f, 0x16, 0x54, + 0x4e, 0x53, 0xd1, 0x0b, 0x65, 0x0d, 0xb2, 0xe2, 0xdb, 0x66, 0xb1, 0xec, 0xfa, 0x6b, 0x2b, 0x77, + 0xff, 0x36, 0x7f, 0x05, 0x6a, 0xee, 0x48, 0x84, 0x39, 0xe6, 0x15, 0x5e, 0x1c, 0x40, 0xe6, 0xc4, + 0x12, 0xfd, 0x57, 0x38, 0x77, 0x40, 0x39, 0x3d, 0xc8, 0x2a, 0xdc, 0x85, 0x64, 0x56, 0x3b, 0x1d, + 0x77, 0x7b, 0xd7, 0xed, 0x56, 0x2c, 0xa1, 0x5a, 0xdd, 0xde, 0x7e, 0xdc, 0x73, 0xbb, 0x95, 0x45, + 0x54, 0x80, 0x8c, 0xeb, 0x79, 0x5b, 0x5e, 0x25, 0x25, 0xac, 0xba, 0x6e, 0xe7, 0x71, 0x6f, 0xd3, + 0xed, 0x56, 0xd2, 0x8f, 0xd2, 0xf9, 0x54, 0x25, 0xed, 0x7c, 0x67, 0xc1, 0xf5, 0x0e, 0x0d, 0xf7, + 0x3a, 0x63, 0xd1, 0x8c, 0x1d, 0x1a, 0x72, 0xf2, 0x35, 0x47, 0xf7, 0x00, 0xc4, 0xbd, 0x03, 0x87, + 0x43, 0xb3, 0x23, 0x0b, 0xed, 0xeb, 0x7a, 0x47, 0x16, 0x3a, 0x4a, 0xd3, 0xeb, 0x7a, 0x05, 0x6d, + 0x24, 0xef, 0x35, 0xb9, 0x08, 0x1f, 0x4c, 0x28, 0x56, 0x77, 0xb7, 0x92, 0x67, 0x5e, 0x51, 0x17, + 0x72, 0xff, 0x7c, 0x6f, 0x19, 0xe8, 0xca, 0x2b, 0x0b, 0x0a, 0x1b, 0xb3, 0x09, 0x0f, 0xc4, 0xf0, + 0xa1, 0x09, 0x54, 0x12, 0x43, 0xa8, 0xf6, 0xc1, 0xdd, 0xcb, 0x4d, 0xaa, 0xb0, 0xad, 0xde, 0xb9, + 0xdc, 0xd2, 0x73, 0x16, 0x1a, 0xd6, 0x3d, 0x0b, 0x3d, 0x83, 0x92, 0x50, 0x1a, 0x06, 0x91, 0xf3, + 0xe6, 0xf6, 0xae, 0xde, 0xba, 0x44, 0x0b, 0x28, 0xf7, 0xed, 0xf7, 0x0f, 0x7f, 0xad, 0x2d, 0x1c, + 0x1e, 0xd7, 0xac, 0x1f, 0x8f, 0x6b, 0xd6, 0xab, 0xe3, 0x9a, 0xf5, 0xcb, 0x71, 0xcd, 0xfa, 0xe6, + 0xb7, 0xda, 0xc2, 0x97, 0x39, 0x8d, 0xfc, 0x33, 0x00, 0x00, 0xff, 0xff, 0xde, 0x5c, 0x11, 0x4b, + 0x3f, 0x0c, 0x00, 0x00, } diff --git a/pkg/storage/raft.proto b/pkg/storage/raft.proto index 6a9faa355c01..9457f95a67c0 100644 --- a/pkg/storage/raft.proto +++ b/pkg/storage/raft.proto @@ -148,6 +148,17 @@ message SnapshotRequest { // The strategy of the snapshot. optional Strategy strategy = 7 [(gogoproto.nullable) = false]; + + // Whether the snapshot uses the unreplicated RaftTruncatedState or not. + // This is generally always true at 2.2 and above outside of the migration + // phase, though theoretically it could take a long time for all ranges + // to update to the new mechanism. This bool is true iff the Raft log at + // the snapshot's applied index is using the new key. In particular, it + // is true if the index itself carries out the migration (in which case + // the data in the snapshot contains neither key). + // + // See VersionUnreplicatedRaftTruncatedState. + optional bool unreplicated_truncated_state = 8 [(gogoproto.nullable) = false]; } optional Header header = 1; diff --git a/pkg/storage/replica_command.go b/pkg/storage/replica_command.go index 22996086e46e..c25de5761740 100644 --- a/pkg/storage/replica_command.go +++ b/pkg/storage/replica_command.go @@ -32,6 +32,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/storage/storagepb" "github.com/cockroachdb/cockroach/pkg/util/causer" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" + "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/retry" @@ -898,8 +899,29 @@ func (r *Replica) sendSnapshot( return &benignError{errors.New("raft status not initialized")} } + // TODO(tbg): send snapshots without the past raft log. This means replacing + // the state's truncated state with one whose index and term equal that of + // the RaftAppliedIndex of the snapshot. It looks like the code sending out + // the actual entries will do the right thing from then on (see anchor + // below). + _ = (*kvBatchSnapshotStrategy)(nil).Send + usesReplicatedTruncatedState, err := engine.MVCCGetProto( + ctx, snap.EngineSnap, keys.RaftTruncatedStateLegacyKey(r.RangeID), hlc.Timestamp{}, nil, engine.MVCCGetOptions{}, + ) + if err != nil { + return errors.Wrap(err, "loading legacy truncated state") + } + req := SnapshotRequest_Header{ State: snap.State, + // Tell the recipient whether it needs to synthesize the new + // unreplicated TruncatedState. It could tell by itself by peeking into + // the data, but it uses a write only batch for performance which + // doesn't support that; this is easier. Notably, this is true if the + // snap index itself is the one at which the migration happens. + // + // See VersionUnreplicatedRaftTruncatedState. + UnreplicatedTruncatedState: !usesReplicatedTruncatedState, RaftMessageRequest: RaftMessageRequest{ RangeID: r.RangeID, FromReplica: fromRepDesc, diff --git a/pkg/storage/replica_raft.go b/pkg/storage/replica_raft.go index dd2541ee8ab6..13f34b45c9b9 100644 --- a/pkg/storage/replica_raft.go +++ b/pkg/storage/replica_raft.go @@ -2241,7 +2241,16 @@ func (r *Replica) applyRaftCommand( oldRaftAppliedIndex, raftAppliedIndex) } - batch := r.store.Engine().NewWriteOnlyBatch() + haveTruncatedState := rResult.State != nil && rResult.State.TruncatedState != nil + var batch engine.Batch + if !haveTruncatedState { + batch = r.store.Engine().NewWriteOnlyBatch() + } else { + // When we update the truncated state, we may need to read the batch + // and can't use a WriteOnlyBatch. This is fine since log truncations + // are tiny batches. + batch = r.store.Engine().NewBatch() + } defer batch.Close() if writeBatch != nil { @@ -2251,8 +2260,7 @@ func (r *Replica) applyRaftCommand( } // The only remaining use of the batch is for range-local keys which we know - // have not been previously written within this batch. Currently the only - // remaining writes are the raft applied index and the updated MVCC stats. + // have not been previously written within this batch. writer := batch.Distinct() // Special-cased MVCC stats handling to exploit commutativity of stats delta @@ -2304,30 +2312,27 @@ func (r *Replica) applyRaftCommand( } } - if rResult.State != nil && rResult.State.TruncatedState != nil { - newTruncatedState := rResult.State.TruncatedState - - // Truncate the Raft log from the entry after the previous - // truncation index to the new truncation index. This is performed - // atomically with the raft command application so that the - // TruncatedState index is always consistent with the state of the - // Raft log itself. We can use the distinct writer because we know - // all writes will be to distinct keys. - // - // Intentionally don't use range deletion tombstones (ClearRange()) - // due to performance concerns connected to having many range - // deletion tombstones. There is a chance that ClearRange will - // perform well here because the tombstones could be "collapsed", - // but it is hardly worth the risk at this point. - prefixBuf := &r.raftMu.stateLoader.RangeIDPrefixBuf - for idx := oldTruncatedState.Index + 1; idx <= newTruncatedState.Index; idx++ { - // NB: RangeIDPrefixBufs have sufficient capacity (32 bytes) to - // avoid allocating when constructing Raft log keys (16 bytes). - unsafeKey := prefixBuf.RaftLogKey(idx) - if err := writer.Clear(engine.MakeMVCCMetadataKey(unsafeKey)); err != nil { - err = errors.Wrapf(err, "unable to clear truncated Raft entries for %+v", newTruncatedState) - return enginepb.MVCCStats{}, err - } + if haveTruncatedState { + apply, err := handleTruncatedStateBelowRaft(ctx, oldTruncatedState, rResult.State.TruncatedState, r.raftMu.stateLoader, writer) + if err != nil { + return enginepb.MVCCStats{}, err + } + if !apply { + // TODO(tbg): As written, there is low confidence that nil'ing out + // the truncated state has the desired effect as our caller actually + // applies the side effects. It may have taken a copy and won't + // observe what we did. + // + // It's very difficult to test this functionality because of how + // difficult it is to test (*Replica).processRaftCommand (and this + // method). Instead of adding yet another terrible that that bends + // reality to its will in some clunky way, assert that we're never + // hitting this branch, which is supposed to be true until we stop + // sending the raft log in snapshots (#34287). + // Morally we would want to drop the command in checkForcedErrLocked, + // but that may be difficult to achieve. + log.Fatal(ctx, log.Safe(fmt.Sprintf("TruncatedState regressed:\nold: %+v\nnew: %+v", oldTruncatedState, rResult.State.TruncatedState))) + rResult.State.TruncatedState = nil } } @@ -2370,3 +2375,79 @@ func (r *Replica) applyRaftCommand( r.store.metrics.RaftCommandCommitLatency.RecordValue(elapsed.Nanoseconds()) return deltaStats, nil } + +func handleTruncatedStateBelowRaft( + ctx context.Context, + oldTruncatedState, newTruncatedState *roachpb.RaftTruncatedState, + loader stateloader.StateLoader, + distinctEng engine.ReadWriter, +) (_apply bool, _ error) { + // If this is a log truncation, load the resulting unreplicated or legacy + // replicated truncated state (in that order). If the migration is happening + // in this command, the result will be an empty message. In steady state + // after the migration, it's the unreplicated truncated state not taking + // into account the current truncation (since the key is unreplicated). + // Either way, we'll update it below. + // + // See VersionUnreplicatedRaftTruncatedState for details. + truncStatePostApply, truncStateIsLegacy, err := loader.LoadRaftTruncatedState(ctx, distinctEng) + if err != nil { + return false, errors.Wrap(err, "loading truncated state") + } + + // Truncate the Raft log from the entry after the previous + // truncation index to the new truncation index. This is performed + // atomically with the raft command application so that the + // TruncatedState index is always consistent with the state of the + // Raft log itself. We can use the distinct writer because we know + // all writes will be to distinct keys. + // + // Intentionally don't use range deletion tombstones (ClearRange()) + // due to performance concerns connected to having many range + // deletion tombstones. There is a chance that ClearRange will + // perform well here because the tombstones could be "collapsed", + // but it is hardly worth the risk at this point. + prefixBuf := &loader.RangeIDPrefixBuf + for idx := oldTruncatedState.Index + 1; idx <= newTruncatedState.Index; idx++ { + // NB: RangeIDPrefixBufs have sufficient capacity (32 bytes) to + // avoid allocating when constructing Raft log keys (16 bytes). + unsafeKey := prefixBuf.RaftLogKey(idx) + if err := distinctEng.Clear(engine.MakeMVCCMetadataKey(unsafeKey)); err != nil { + return false, errors.Wrapf(err, "unable to clear truncated Raft entries for %+v", newTruncatedState) + } + } + + if !truncStateIsLegacy { + if truncStatePostApply.Index < newTruncatedState.Index { + // There are two cases here (though handled just the same). In the + // first case, the Raft command has just deleted the legacy + // replicated truncated state key as part of the migration (so + // truncStateIsLegacy is now false for the first time and + // truncStatePostApply is zero) and we need to atomically write the + // new, unreplicated, key. Or we've already migrated earlier, in + // which case truncStatePostApply equals the current value of the + // new key (which wasn't touched by the batch), and we need to + // overwrite it if this truncation "moves it forward". + + // NB: before the first log truncation evaluated under the + // cluster version which activates this code (see anchor below) this + // block is never reached as truncStatePostApply will equal newTruncatedState. + _ = cluster.VersionUnreplicatedRaftTruncatedState + + if err := engine.MVCCPutProto( + ctx, distinctEng, nil /* ms */, prefixBuf.RaftTruncatedStateKey(), + hlc.Timestamp{}, nil /* txn */, newTruncatedState, + ); err != nil { + return false, errors.Wrap(err, "unable to migrate RaftTruncatedState") + } + // Have migrated and this new truncated state is moving us forward. + // Tell caller that we applied it and that so should they. + return true, nil + } + // Have migrated, but this truncated state moves the existing one + // backwards, so instruct caller to not update in-memory state. + return false, nil + } + // Haven't migrated yet, don't ever discard the update. + return true, nil +} diff --git a/pkg/storage/replica_raft_truncation_test.go b/pkg/storage/replica_raft_truncation_test.go new file mode 100644 index 000000000000..81caed41acc4 --- /dev/null +++ b/pkg/storage/replica_raft_truncation_test.go @@ -0,0 +1,119 @@ +// Copyright 2019 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package storage + +import ( + "bytes" + "context" + "fmt" + "testing" + + "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/engine" + "github.com/cockroachdb/cockroach/pkg/storage/stateloader" + "github.com/cockroachdb/cockroach/pkg/testutils/datadriven" + "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/stretchr/testify/assert" +) + +func TestHandleTruncatedStateBelowRaft(t *testing.T) { + defer leaktest.AfterTest(t)() + + // This test verifies the expected behavior of the downstream-of-Raft log + // truncation code, in particular regarding the migration below: + _ = cluster.VersionUnreplicatedRaftTruncatedState + + ctx := context.Background() + + // neither exists (migration) + // old one exists (no migration) + // new one exists (migrated already) + // truncstate regresses + + var prevTruncatedState roachpb.RaftTruncatedState + datadriven.Walk(t, "testdata/truncated_state_migration", func(t *testing.T, path string) { + const rangeID = 12 + loader := stateloader.Make(rangeID) + eng := engine.NewInMem(roachpb.Attributes{}, 1<<20) + defer eng.Close() + + datadriven.RunTest(t, path, func(d *datadriven.TestData) string { + switch d.Cmd { + case "prev": + d.ScanArgs(t, "index", &prevTruncatedState.Index) + d.ScanArgs(t, "term", &prevTruncatedState.Term) + return "" + case "put": + var index uint64 + var term uint64 + var legacy bool + d.ScanArgs(t, "index", &index) + d.ScanArgs(t, "term", &term) + d.ScanArgs(t, "legacy", &legacy) + + truncState := &roachpb.RaftTruncatedState{ + Index: index, + Term: term, + } + + if legacy { + assert.NoError(t, loader.SetLegacyRaftTruncatedState(ctx, eng, nil, truncState)) + } else { + assert.NoError(t, loader.SetRaftTruncatedState(ctx, eng, truncState)) + } + return "" + case "handle": + var buf bytes.Buffer + + var index uint64 + var term uint64 + d.ScanArgs(t, "index", &index) + d.ScanArgs(t, "term", &term) + + newTruncatedState := &roachpb.RaftTruncatedState{ + Index: index, + Term: term, + } + + apply, err := handleTruncatedStateBelowRaft(ctx, &prevTruncatedState, newTruncatedState, loader, eng) + if err != nil { + return err.Error() + } + fmt.Fprintf(&buf, "apply: %t\n", apply) + + for _, key := range []roachpb.Key{ + keys.RaftTruncatedStateLegacyKey(rangeID), + keys.RaftTruncatedStateKey(rangeID), + } { + var truncatedState roachpb.RaftTruncatedState + ok, err := engine.MVCCGetProto(ctx, eng, key, hlc.Timestamp{}, &truncatedState, engine.MVCCGetOptions{}) + if err != nil { + t.Fatal(err) + } + if !ok { + continue + } + fmt.Fprintf(&buf, "%s -> index=%d term=%d\n", key, truncatedState.Index, truncatedState.Term) + } + return buf.String() + default: + } + return fmt.Sprintf("unsupported: %s", d.Cmd) + }) + }) +} diff --git a/pkg/storage/replica_raftstorage.go b/pkg/storage/replica_raftstorage.go index 4e1a6b88eb7c..695942ae56b3 100644 --- a/pkg/storage/replica_raftstorage.go +++ b/pkg/storage/replica_raftstorage.go @@ -212,7 +212,7 @@ func entries( } // No results, was it due to unavailability or truncation? - ts, err := rsl.LoadLegacyRaftTruncatedState(ctx, e) + ts, _, err := rsl.LoadRaftTruncatedState(ctx, e) if err != nil { return nil, err } @@ -281,7 +281,7 @@ func term( // sideloaded entries. We only need the term, so this is what we do. ents, err := entries(ctx, rsl, eng, rangeID, eCache, nil /* sideloaded */, i, i+1, math.MaxUint64 /* maxBytes */) if err == raft.ErrCompacted { - ts, err := rsl.LoadLegacyRaftTruncatedState(ctx, eng) + ts, _, err := rsl.LoadRaftTruncatedState(ctx, eng) if err != nil { return 0, err } @@ -318,7 +318,7 @@ func (r *Replica) raftTruncatedStateLocked( if r.mu.state.TruncatedState != nil { return *r.mu.state.TruncatedState, nil } - ts, err := r.mu.stateLoader.LoadLegacyRaftTruncatedState(ctx, r.store.Engine()) + ts, _, err := r.mu.stateLoader.LoadRaftTruncatedState(ctx, r.store.Engine()) if err != nil { return ts, err } @@ -488,8 +488,17 @@ type IncomingSnapshot struct { // The Raft log entries for this snapshot. LogEntries [][]byte // The replica state at the time the snapshot was generated (never nil). - State *storagepb.ReplicaState - snapType string + State *storagepb.ReplicaState + // + // When true, this snapshot contains an unreplicated TruncatedState. When + // false, the TruncatedState is replicated (see the reference below) and the + // recipient must avoid also writing the unreplicated TruncatedState. The + // migration to an unreplicated TruncatedState will be carried out during + // the next log truncation (assuming cluster version is bumped at that + // point). + // See the comment on VersionUnreplicatedRaftTruncatedState for details. + UsesUnreplicatedTruncatedState bool + snapType string } // snapshot creates an OutgoingSnapshot containing a rocksdb snapshot for the @@ -861,6 +870,19 @@ func (r *Replica) applySnapshot( distinctBatch := batch.Distinct() stats.batch = timeutil.Now() + if inSnap.UsesUnreplicatedTruncatedState { + // We're using the unreplicated truncated state, which we need to + // manually persist to disk. If we're not taking this branch, the + // snapshot contains a legacy TruncatedState and we don't need to do + // anything (in fact, must not -- the invariant is that exactly one of + // them exists at any given point in the state machine). + if err := stateloader.Make(s.Desc.RangeID).SetRaftTruncatedState( + ctx, distinctBatch, s.TruncatedState, + ); err != nil { + return err + } + } + logEntries := make([]raftpb.Entry, len(inSnap.LogEntries)) for i, bytes := range inSnap.LogEntries { if err := protoutil.Unmarshal(bytes, &logEntries[i]); err != nil { diff --git a/pkg/storage/replica_test.go b/pkg/storage/replica_test.go index 1467bdc3d855..66752beac9bb 100644 --- a/pkg/storage/replica_test.go +++ b/pkg/storage/replica_test.go @@ -257,6 +257,7 @@ func (tc *testContext) StartWithStoreConfig(t testing.TB, stopper *stop.Stopper, hlc.Timestamp{}, hlc.Timestamp{}, bootstrapVersion.Version, + stateloader.TruncatedStateUnreplicated, ); err != nil { t.Fatal(err) } @@ -9829,7 +9830,10 @@ func TestReplicaBootstrapRangeAppliedStateKey(t *testing.T) { // Save the ReplicaState and perform persistent assertions again. repl.raftMu.Lock() repl.mu.Lock() - if _, err := repl.mu.stateLoader.Save(ctx, tc.engine, repl.mu.state); err != nil { + if _, err := repl.mu.stateLoader.Save( + ctx, tc.engine, repl.mu.state, + stateloader.TruncatedStateUnreplicated, + ); err != nil { t.Fatalf("could not save ReplicaState: %v", err) } repl.mu.Unlock() diff --git a/pkg/storage/stateloader/initial.go b/pkg/storage/stateloader/initial.go index 3027c5a481b5..aaaa58764b71 100644 --- a/pkg/storage/stateloader/initial.go +++ b/pkg/storage/stateloader/initial.go @@ -55,8 +55,14 @@ func WriteInitialReplicaState( gcThreshold hlc.Timestamp, txnSpanGCThreshold hlc.Timestamp, activeVersion roachpb.Version, + truncStateType TruncatedStateType, ) (enginepb.MVCCStats, error) { rsl := Make(desc.RangeID) + // NB: be careful using activeVersion here. One caller of this code is the + // split trigger, and the version with which the split trigger is called can + // vary across followers. Thus, actions which require coordination cannot + // use the version as a trigger (this is why this method takes a + // truncStateType argument). var s storagepb.ReplicaState s.TruncatedState = &roachpb.RaftTruncatedState{ @@ -101,7 +107,7 @@ func WriteInitialReplicaState( log.Fatalf(ctx, "expected trivial TxnSpanGCThreshold, but found %+v", existingTxnSpanGCThreshold) } - newMS, err := rsl.Save(ctx, eng, s) + newMS, err := rsl.Save(ctx, eng, s, truncStateType) if err != nil { return enginepb.MVCCStats{}, err } @@ -125,9 +131,10 @@ func WriteInitialState( gcThreshold hlc.Timestamp, txnSpanGCThreshold hlc.Timestamp, bootstrapVersion roachpb.Version, + truncStateType TruncatedStateType, ) (enginepb.MVCCStats, error) { newMS, err := WriteInitialReplicaState( - ctx, eng, ms, desc, lease, gcThreshold, txnSpanGCThreshold, bootstrapVersion) + ctx, eng, ms, desc, lease, gcThreshold, txnSpanGCThreshold, bootstrapVersion, truncStateType) if err != nil { return enginepb.MVCCStats{}, err } diff --git a/pkg/storage/stateloader/stateloader.go b/pkg/storage/stateloader/stateloader.go index 995b19311136..14f43cc36070 100644 --- a/pkg/storage/stateloader/stateloader.go +++ b/pkg/storage/stateloader/stateloader.go @@ -105,7 +105,7 @@ func (rsl StateLoader) Load( // The truncated state should not be optional (i.e. the pointer is // pointless), but it is and the migration is not worth it. - truncState, err := rsl.LoadLegacyRaftTruncatedState(ctx, reader) + truncState, _, err := rsl.LoadRaftTruncatedState(ctx, reader) if err != nil { return storagepb.ReplicaState{}, err } @@ -114,6 +114,17 @@ func (rsl StateLoader) Load( return s, nil } +// TruncatedStateType determines whether to use a replicated (legacy) or an +// unreplicated TruncatedState. See VersionUnreplicatedRaftTruncatedStateKey. +type TruncatedStateType int + +const ( + // TruncatedStateLegacyReplicated means use the legacy (replicated) key. + TruncatedStateLegacyReplicated TruncatedStateType = iota + // TruncatedStateUnreplicated means use the new (unreplicated) key. + TruncatedStateUnreplicated +) + // Save persists the given ReplicaState to disk. It assumes that the contained // Stats are up-to-date and returns the stats which result from writing the // updated State. @@ -126,7 +137,10 @@ func (rsl StateLoader) Load( // missing whenever save is called. Optional values should be reserved // strictly for use in Result. Do before merge. func (rsl StateLoader) Save( - ctx context.Context, eng engine.ReadWriter, state storagepb.ReplicaState, + ctx context.Context, + eng engine.ReadWriter, + state storagepb.ReplicaState, + truncStateType TruncatedStateType, ) (enginepb.MVCCStats, error) { ms := state.Stats if err := rsl.SetLease(ctx, eng, ms, *state.Lease); err != nil { @@ -138,8 +152,14 @@ func (rsl StateLoader) Save( if err := rsl.SetTxnSpanGCThreshold(ctx, eng, ms, state.TxnSpanGCThreshold); err != nil { return enginepb.MVCCStats{}, err } - if err := rsl.SetLegacyRaftTruncatedState(ctx, eng, ms, state.TruncatedState); err != nil { - return enginepb.MVCCStats{}, err + if truncStateType == TruncatedStateLegacyReplicated { + if err := rsl.SetLegacyRaftTruncatedState(ctx, eng, ms, state.TruncatedState); err != nil { + return enginepb.MVCCStats{}, err + } + } else { + if err := rsl.SetRaftTruncatedState(ctx, eng, state.TruncatedState); err != nil { + return enginepb.MVCCStats{}, err + } } if state.UsingAppliedStateKey { rai, lai := state.RaftAppliedIndex, state.LeaseAppliedIndex @@ -416,19 +436,6 @@ func (rsl StateLoader) SetMVCCStats( return rsl.writeLegacyMVCCStatsInternal(ctx, eng, newMS) } -// LoadLegacyRaftTruncatedState loads the truncated state. -func (rsl StateLoader) LoadLegacyRaftTruncatedState( - ctx context.Context, reader engine.Reader, -) (roachpb.RaftTruncatedState, error) { - var truncState roachpb.RaftTruncatedState - if _, err := engine.MVCCGetProto( - ctx, reader, rsl.RaftTruncatedStateLegacyKey(), hlc.Timestamp{}, &truncState, engine.MVCCGetOptions{}, - ); err != nil { - return roachpb.RaftTruncatedState{}, err - } - return truncState, nil -} - // SetLegacyRaftTruncatedState overwrites the truncated state. func (rsl StateLoader) SetLegacyRaftTruncatedState( ctx context.Context, @@ -508,7 +515,7 @@ func (rsl StateLoader) LoadLastIndex(ctx context.Context, reader engine.Reader) if lastIndex == 0 { // The log is empty, which means we are either starting from scratch // or the entire log has been truncated away. - lastEnt, err := rsl.LoadLegacyRaftTruncatedState(ctx, reader) + lastEnt, _, err := rsl.LoadRaftTruncatedState(ctx, reader) if err != nil { return 0, err } @@ -517,6 +524,48 @@ func (rsl StateLoader) LoadLastIndex(ctx context.Context, reader engine.Reader) return lastIndex, nil } +// LoadRaftTruncatedState loads the truncated state. The returned boolean returns +// whether the result was read from the TruncatedStateLegacyKey. If both keys +// are missing, it is false which is used to migrate into the unreplicated key. +// +// See VersionUnreplicatedRaftTruncatedState. +func (rsl StateLoader) LoadRaftTruncatedState( + ctx context.Context, reader engine.Reader, +) (_ roachpb.RaftTruncatedState, isLegacy bool, _ error) { + var truncState roachpb.RaftTruncatedState + if found, err := engine.MVCCGetProto( + ctx, reader, rsl.RaftTruncatedStateKey(), hlc.Timestamp{}, &truncState, engine.MVCCGetOptions{}, + ); err != nil { + return roachpb.RaftTruncatedState{}, false, err + } else if found { + return truncState, false, nil + } + + // If the "new" truncated state isn't there (yet), fall back to the legacy + // truncated state. The next log truncation will atomically rewrite them + // assuming the cluster version has advanced sufficiently. + // + // See VersionUnreplicatedRaftTruncatedState. + legacyFound, err := engine.MVCCGetProto( + ctx, reader, rsl.RaftTruncatedStateLegacyKey(), hlc.Timestamp{}, &truncState, engine.MVCCGetOptions{}, + ) + if err != nil { + return roachpb.RaftTruncatedState{}, false, err + } + return truncState, legacyFound, nil +} + +// SetRaftTruncatedState overwrites the truncated state. +func (rsl StateLoader) SetRaftTruncatedState( + ctx context.Context, eng engine.ReadWriter, truncState *roachpb.RaftTruncatedState, +) error { + if (*truncState == roachpb.RaftTruncatedState{}) { + return errors.New("cannot persist empty RaftTruncatedState") + } + return engine.MVCCPutProto(ctx, eng, nil, /* ms */ + rsl.RaftTruncatedStateKey(), hlc.Timestamp{}, nil, truncState) +} + // LoadReplicaDestroyedError loads the replica destroyed error for the specified // range. If there is no error, nil is returned. func (rsl StateLoader) LoadReplicaDestroyedError( @@ -575,7 +624,7 @@ func (rsl StateLoader) SynthesizeRaftState(ctx context.Context, eng engine.ReadW if err != nil { return err } - truncState, err := rsl.LoadLegacyRaftTruncatedState(ctx, eng) + truncState, _, err := rsl.LoadRaftTruncatedState(ctx, eng) if err != nil { return err } diff --git a/pkg/storage/stats_test.go b/pkg/storage/stats_test.go index 050170c1593a..fd1bc16af872 100644 --- a/pkg/storage/stats_test.go +++ b/pkg/storage/stats_test.go @@ -31,8 +31,8 @@ import ( // writeInitialState(). func initialStats() enginepb.MVCCStats { return enginepb.MVCCStats{ - SysBytes: 130, - SysCount: 4, + SysBytes: 98, + SysCount: 3, } } func TestRangeStatsEmpty(t *testing.T) { diff --git a/pkg/storage/store.go b/pkg/storage/store.go index c5497e144017..79268f0e121a 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -2174,12 +2174,25 @@ func (s *Store) WriteInitialData( } } + // See the cluster version for more details. We're basically saying that if the cluster + // is bootstrapped at a version that uses the unreplicated truncated state, initialize + // it with such a truncated state. + truncStateType := stateloader.TruncatedStateUnreplicated + if bootstrapVersion.Less(cluster.VersionByKey(cluster.VersionUnreplicatedRaftTruncatedState)) { + truncStateType = stateloader.TruncatedStateLegacyReplicated + } + lease := roachpb.BootstrapLease() _, err := stateloader.WriteInitialState( ctx, batch, enginepb.MVCCStats{}, *desc, - lease, hlc.Timestamp{}, hlc.Timestamp{}, bootstrapVersion) + lease, + hlc.Timestamp{}, + hlc.Timestamp{}, + bootstrapVersion, + truncStateType, + ) if err != nil { return err } diff --git a/pkg/storage/store_snapshot.go b/pkg/storage/store_snapshot.go index e906b82edec2..b14202e65520 100644 --- a/pkg/storage/store_snapshot.go +++ b/pkg/storage/store_snapshot.go @@ -140,11 +140,12 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( } inSnap := IncomingSnapshot{ - SnapUUID: snapUUID, - Batches: batches, - LogEntries: logEntries, - State: &header.State, - snapType: snapTypeRaft, + UsesUnreplicatedTruncatedState: header.UnreplicatedTruncatedState, + SnapUUID: snapUUID, + Batches: batches, + LogEntries: logEntries, + State: &header.State, + snapType: snapTypeRaft, } if header.RaftMessageRequest.ToReplica.ReplicaID == 0 { inSnap.snapType = snapTypePreemptive diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index b63285db8b7d..d8c81ebdd8b7 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -1348,6 +1348,7 @@ func splitTestRange(store *Store, key, splitKey roachpb.RKey, t *testing.T) *Rep context.Background(), store.engine, enginepb.MVCCStats{}, *desc, roachpb.Lease{}, hlc.Timestamp{}, hlc.Timestamp{}, store.ClusterSettings().Version.Version().Version, + stateloader.TruncatedStateUnreplicated, ); err != nil { t.Fatal(err) } @@ -2778,6 +2779,7 @@ func TestStoreRemovePlaceholderOnRaftIgnored(t *testing.T) { ctx, s.Engine(), enginepb.MVCCStats{}, *repl1.Desc(), roachpb.Lease{}, hlc.Timestamp{}, hlc.Timestamp{}, s.ClusterSettings().Version.Version().Version, + stateloader.TruncatedStateUnreplicated, ); err != nil { t.Fatal(err) } diff --git a/pkg/storage/testdata/truncated_state_migration/migration b/pkg/storage/testdata/truncated_state_migration/migration new file mode 100644 index 000000000000..d2c6ea1e6d83 --- /dev/null +++ b/pkg/storage/testdata/truncated_state_migration/migration @@ -0,0 +1,25 @@ +# Migrating after VersionUnreplicatedRaftTruncatedState was enabled. During the +# migration, no TruncatedState is on disk, but we write the new, unreplicated, +# one (note the /u/ in the key) + +prev index=100 term=9 +---- + +handle index=150 term=9 +---- +apply: true +/Local/RangeID/12/u/RaftTruncatedState -> index=150 term=9 + +# Simulate another truncation that moves forward. + +handle index=170 term=9 +---- +apply: true +/Local/RangeID/12/u/RaftTruncatedState -> index=170 term=9 + +# ... and one that moves backwards and should not take effect. + +handle index=150 term=9 +---- +apply: false +/Local/RangeID/12/u/RaftTruncatedState -> index=170 term=9 diff --git a/pkg/storage/testdata/truncated_state_migration/pre_migration b/pkg/storage/testdata/truncated_state_migration/pre_migration new file mode 100644 index 000000000000..e84177bd0b51 --- /dev/null +++ b/pkg/storage/testdata/truncated_state_migration/pre_migration @@ -0,0 +1,26 @@ +# Mode of operation below VersionUnreplicatedRaftTruncatedState. +# We don't mess with the on-disk state nor do we ever drop updates. + +prev index=100 term=9 +---- + +put legacy=true index=100 term=9 +---- + +handle index=100 term=9 +---- +apply: true +/Local/RangeID/12/r/RaftTruncatedState -> index=100 term=9 + +# Note that the below aren't actually possible in practice +# as a divergence won't happen before the migration. + +handle index=150 term=9 +---- +apply: true +/Local/RangeID/12/r/RaftTruncatedState -> index=100 term=9 + +handle index=60 term=9 +---- +apply: true +/Local/RangeID/12/r/RaftTruncatedState -> index=100 term=9 From 0c36412a8ef46ad9fb7e5290c493ebbf8efad736 Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Mon, 11 Feb 2019 22:39:06 +0100 Subject: [PATCH 10/10] storage: improve message on slow Raft proposal Release note: None --- pkg/storage/replica_write.go | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/pkg/storage/replica_write.go b/pkg/storage/replica_write.go index 021159252437..ef4aa58ad787 100644 --- a/pkg/storage/replica_write.go +++ b/pkg/storage/replica_write.go @@ -221,16 +221,33 @@ func (r *Replica) tryExecuteWriteBatch( return propResult.Reply, propResult.Err, propResult.ProposalRetry case <-slowTimer.C: slowTimer.Read = true - log.Warningf(ctx, "have been waiting %s for proposing command %s", - base.SlowRequestThreshold, ba) + log.Warningf(ctx, `have been waiting %.2fs for proposing command %s. +This range is likely unavailable. +Please submit this message at + + https://github.com/cockroachdb/cockroach/issues/new/choose + +along with + + https://yourhost:8080/#/reports/range/%d + +and the following Raft status: %+v`, + timeutil.Since(tBegin).Seconds(), + ba, + r.RangeID, + r.RaftStatus(), + ) r.store.metrics.SlowRaftRequests.Inc(1) defer func() { r.store.metrics.SlowRaftRequests.Dec(1) - contextStr := "" - if err := ctx.Err(); err != nil { - contextStr = " with context cancellation" - } - log.Infof(ctx, "slow command %s finished after %s%s", ba, timeutil.Since(tBegin), contextStr) + log.Infof( + ctx, + "slow command %s finished after %.2fs with error %v, retry %d", + ba, + timeutil.Since(tBegin).Seconds(), + pErr, + retry, + ) }() case <-ctxDone: