From 7ac3459dc36fb38881c5a76079be7991c2040996 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 9 May 2022 15:11:34 +0800 Subject: [PATCH 01/14] feat(planner): support in new planner --- query/src/sql/exec/expression_builder.rs | 17 +++++ query/src/sql/exec/mod.rs | 72 +++++++++++++++++++ query/src/sql/planner/binder/bind_context.rs | 8 ++- query/src/sql/planner/binder/mod.rs | 1 + query/src/sql/planner/binder/project.rs | 8 ++- .../src/sql/planner/binder/scalar_visitor.rs | 7 +- query/src/sql/planner/binder/select.rs | 8 ++- query/src/sql/planner/binder/sort.rs | 57 +++++++++++++++ query/src/sql/planner/plans/mod.rs | 4 ++ query/src/sql/planner/plans/scalar.rs | 20 ++++++ query/src/sql/planner/plans/sort.rs | 67 +++++++++++++++++ .../20+_others/20_0001_planner_v2.result | 38 +++++++++- .../20+_others/20_0001_planner_v2.sql | 12 +++- 13 files changed, 309 insertions(+), 10 deletions(-) create mode 100644 query/src/sql/planner/binder/sort.rs create mode 100644 query/src/sql/planner/plans/sort.rs diff --git a/query/src/sql/exec/expression_builder.rs b/query/src/sql/exec/expression_builder.rs index bfeaadbdeca1..f9a3d704ba4d 100644 --- a/query/src/sql/exec/expression_builder.rs +++ b/query/src/sql/exec/expression_builder.rs @@ -25,6 +25,7 @@ use crate::sql::plans::ComparisonExpr; use crate::sql::plans::ConstantExpr; use crate::sql::plans::FunctionCall; use crate::sql::plans::OrExpr; +use crate::sql::plans::OrderExpr; use crate::sql::plans::Scalar; use crate::sql::IndexType; use crate::sql::Metadata; @@ -108,6 +109,22 @@ impl<'a> ExpressionBuilder<'a> { pg_style: false, }) } + Scalar::Order(OrderExpr { + expr, + asc, + nulls_first, + }) => { + let expr = self.build(expr)?; + let asc = asc.unwrap_or(true); + // NULLS FIRST is the default for DESC order, and NULLS LAST otherwise + let nulls_first = nulls_first.unwrap_or(!asc); + Ok(Expression::Sort { + expr: Box::new(expr.clone()), + asc, + nulls_first, + origin_expr: Box::new(expr), + }) + } } } diff --git a/query/src/sql/exec/mod.rs b/query/src/sql/exec/mod.rs index 3c61f109f0af..c2414ad7b782 100644 --- a/query/src/sql/exec/mod.rs +++ b/query/src/sql/exec/mod.rs @@ -40,11 +40,15 @@ use crate::pipelines::new::processors::HashJoinState; use crate::pipelines::new::processors::ProjectionTransform; use crate::pipelines::new::processors::SinkBuildHashTable; use crate::pipelines::new::processors::Sinker; +use crate::pipelines::new::processors::SortMergeCompactor; use crate::pipelines::new::processors::TransformAggregator; use crate::pipelines::new::processors::TransformFilter; use crate::pipelines::new::processors::TransformHashJoinProbe; +use crate::pipelines::new::processors::TransformSortMerge; +use crate::pipelines::new::processors::TransformSortPartial; use crate::pipelines::new::NewPipeline; use crate::pipelines::new::SinkPipeBuilder; +use crate::pipelines::transforms::get_sort_descriptions; use crate::sessions::QueryContext; use crate::sql::exec::data_schema_builder::DataSchemaBuilder; use crate::sql::exec::expression_builder::ExpressionBuilder; @@ -57,6 +61,7 @@ use crate::sql::plans::PhysicalHashJoin; use crate::sql::plans::PhysicalScan; use crate::sql::plans::PlanType; use crate::sql::plans::ProjectPlan; +use crate::sql::plans::SortPlan; use crate::sql::IndexType; use crate::sql::Metadata; @@ -188,6 +193,12 @@ impl PipelineBuilder { pipeline, ) } + PlanType::Sort => { + let sort_plan: SortPlan = plan.try_into()?; + let input_schema = + self.build_pipeline(context, &expression.children()[0], pipeline)?; + self.build_order_by(&sort_plan, input_schema, pipeline) + } _ => Err(ErrorCode::LogicalError("Invalid physical plan")), } } @@ -482,4 +493,65 @@ impl PipelineBuilder { pipeline.add_pipe(sink_pipeline_builder.finalize()); Ok(()) } + + fn build_order_by( + &mut self, + sort_plan: &SortPlan, + input_schema: DataSchemaRef, + pipeline: &mut NewPipeline, + ) -> Result { + let output_schema = input_schema.clone(); + let eb = ExpressionBuilder::create(&self.metadata); + let mut expressions = Vec::with_capacity(sort_plan.items.len()); + for item in sort_plan.items.iter() { + expressions.push(eb.build(item)?); + } + + //TODO(xudong963): Add rows_limit + + // processor 1: block ---> sort_stream + // processor 2: block ---> sort_stream + // processor 3: block ---> sort_stream + pipeline.add_transform(|transform_input_port, transform_output_port| { + TransformSortPartial::try_create( + transform_input_port, + transform_output_port, + None, + get_sort_descriptions(&input_schema, expressions.as_slice())?, + ) + })?; + + // processor 1: [sorted blocks ...] ---> merge to one sorted block + // processor 2: [sorted blocks ...] ---> merge to one sorted block + // processor 3: [sorted blocks ...] ---> merge to one sorted block + pipeline.add_transform(|transform_input_port, transform_output_port| { + TransformSortMerge::try_create( + transform_input_port, + transform_output_port, + SortMergeCompactor::new( + None, + get_sort_descriptions(&input_schema, expressions.as_slice())?, + ), + ) + })?; + + // processor1 sorted block -- + // \ + // processor2 sorted block ----> processor --> merge to one sorted block + // / + // processor3 sorted block -- + pipeline.resize(1)?; + pipeline.add_transform(|transform_input_port, transform_output_port| { + TransformSortMerge::try_create( + transform_input_port, + transform_output_port, + SortMergeCompactor::new( + None, + get_sort_descriptions(&input_schema, expressions.as_slice())?, + ), + ) + })?; + + Ok(output_schema) + } } diff --git a/query/src/sql/planner/binder/bind_context.rs b/query/src/sql/planner/binder/bind_context.rs index 5b147d3976d1..89fa8161b6d9 100644 --- a/query/src/sql/planner/binder/bind_context.rs +++ b/query/src/sql/planner/binder/bind_context.rs @@ -43,13 +43,18 @@ pub struct ColumnBinding { #[derive(Clone, Default)] pub struct BindContext { _parent: Option>, - columns: Vec, + pub columns: Vec, /// The relational operator in current context pub expression: Option, /// Aggregation scalar expression pub agg_scalar_exprs: Option>, + + /// Order by columnBinding, consider the sql: select a as c from t1 order by c, a, + /// Order by requires not just the columns in the selection, + /// but the columns of the entire table as well as the columns of the selection + pub group_by_columns: Vec, } impl BindContext { @@ -63,6 +68,7 @@ impl BindContext { columns: vec![], expression: None, agg_scalar_exprs: None, + group_by_columns: vec![], } } diff --git a/query/src/sql/planner/binder/mod.rs b/query/src/sql/planner/binder/mod.rs index 47f62cf2b85f..5597ab6ac248 100644 --- a/query/src/sql/planner/binder/mod.rs +++ b/query/src/sql/planner/binder/mod.rs @@ -33,6 +33,7 @@ mod scalar; mod scalar_common; mod scalar_visitor; mod select; +mod sort; /// Binder is responsible to transform AST of a query into a canonical logical SExpr. /// diff --git a/query/src/sql/planner/binder/project.rs b/query/src/sql/planner/binder/project.rs index e00af52e6ca4..21d5797e2de0 100644 --- a/query/src/sql/planner/binder/project.rs +++ b/query/src/sql/planner/binder/project.rs @@ -22,8 +22,10 @@ use crate::sql::planner::binder::scalar::ScalarBinder; use crate::sql::planner::binder::BindContext; use crate::sql::planner::binder::Binder; use crate::sql::planner::binder::ColumnBinding; +use crate::sql::plans::BoundColumnRef; use crate::sql::plans::ProjectItem; use crate::sql::plans::ProjectPlan; +use crate::sql::plans::Scalar; impl Binder { /// Try to build a `ProjectPlan` to satisfy `output_context`. @@ -74,6 +76,7 @@ impl Binder { input_context: &BindContext, ) -> Result { let mut output_context = BindContext::new(); + output_context.group_by_columns = input_context.columns.clone(); output_context.expression = input_context.expression.clone(); for select_target in select_list { match select_target { @@ -121,8 +124,11 @@ impl Binder { column_name: expr_name, index, data_type, - scalar: Some(Box::new(bound_expr)), + scalar: Some(Box::new(bound_expr.clone())), }; + if !matches!(bound_expr, Scalar::BoundColumnRef(BoundColumnRef { .. })) { + output_context.group_by_columns.push(column_binding.clone()); + } output_context.add_column_binding(column_binding); } } diff --git a/query/src/sql/planner/binder/scalar_visitor.rs b/query/src/sql/planner/binder/scalar_visitor.rs index 9286ba19df7a..ca4eb6244cb4 100644 --- a/query/src/sql/planner/binder/scalar_visitor.rs +++ b/query/src/sql/planner/binder/scalar_visitor.rs @@ -20,6 +20,7 @@ use crate::sql::plans::CastExpr; use crate::sql::plans::ComparisonExpr; use crate::sql::plans::FunctionCall; use crate::sql::plans::OrExpr; +use crate::sql::plans::OrderExpr; use crate::sql::plans::Scalar; /// Controls how the visitor recursion should proceed. @@ -74,11 +75,13 @@ pub trait ScalarVisitor: Sized { stack.push(RecursionProcessing::Call(arg)); } } - Scalar::BoundColumnRef(_) => {} - Scalar::ConstantExpr(_) => {} + Scalar::BoundColumnRef(_) | Scalar::ConstantExpr(_) => {} Scalar::Cast(CastExpr { argument, .. }) => { stack.push(RecursionProcessing::Call(argument)) } + Scalar::Order(OrderExpr { expr, .. }) => { + stack.push(RecursionProcessing::Call(expr)) + } } visitor diff --git a/query/src/sql/planner/binder/select.rs b/query/src/sql/planner/binder/select.rs index 7bf5e14ce109..2093d5d24b02 100644 --- a/query/src/sql/planner/binder/select.rs +++ b/query/src/sql/planner/binder/select.rs @@ -49,15 +49,17 @@ impl Binder { query: &Query, bind_context: &BindContext, ) -> Result { - let bind_context = match &query.body { + let mut bind_context = match &query.body { SetExpr::Select(stmt) => self.bind_select_stmt(stmt, bind_context).await, SetExpr::Query(stmt) => self.bind_query(stmt, bind_context).await, _ => Err(ErrorCode::UnImplement("Unsupported query type")), }?; - // TODO: support ORDER BY if !query.order_by.is_empty() { - return Err(ErrorCode::UnImplement("Unsupported ORDER BY")); + let bind_context_cols = bind_context.columns.clone(); + bind_context.columns = bind_context.group_by_columns.clone(); + self.bind_order_by(&query.order_by, &mut bind_context)?; + bind_context.columns = bind_context_cols; } if !query.limit.is_empty() { diff --git a/query/src/sql/planner/binder/sort.rs b/query/src/sql/planner/binder/sort.rs new file mode 100644 index 000000000000..6e95e32796c1 --- /dev/null +++ b/query/src/sql/planner/binder/sort.rs @@ -0,0 +1,57 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use common_ast::ast::OrderByExpr; +use common_datavalues::DataTypeImpl; +use common_exception::Result; + +use crate::sql::binder::scalar::ScalarBinder; +use crate::sql::binder::Binder; +use crate::sql::optimizer::SExpr; +use crate::sql::plans::OrderExpr; +use crate::sql::plans::Scalar; +use crate::sql::plans::Scalar::Order; +use crate::sql::plans::SortPlan; +use crate::sql::BindContext; + +impl Binder { + pub(super) fn bind_order_by( + &mut self, + order_by: &[OrderByExpr], + bind_context: &mut BindContext, + ) -> Result<()> { + let scalar_binder = ScalarBinder::new(bind_context); + let order_by_exprs = order_by + .iter() + .map(|expr| scalar_binder.bind_expr(&expr.expr)) + .collect::>>()?; + + let mut order_by_items = Vec::with_capacity(order_by_exprs.len()); + for (idx, order_by_expr) in order_by_exprs.iter().enumerate() { + let order_by_item = OrderExpr { + expr: Box::new(order_by_expr.0.clone()), + asc: order_by[idx].asc, + nulls_first: order_by[idx].nulls_first, + }; + order_by_items.push(Order(order_by_item)); + } + let sort_plan = SortPlan { + items: order_by_items, + }; + let new_expr = + SExpr::create_unary(sort_plan.into(), bind_context.expression.clone().unwrap()); + bind_context.expression = Some(new_expr); + Ok(()) + } +} diff --git a/query/src/sql/planner/plans/mod.rs b/query/src/sql/planner/plans/mod.rs index 221cd19de24f..4fa4b94f5384 100644 --- a/query/src/sql/planner/plans/mod.rs +++ b/query/src/sql/planner/plans/mod.rs @@ -21,6 +21,7 @@ mod pattern; mod physical_scan; mod project; mod scalar; +mod sort; use std::any::Any; @@ -35,6 +36,7 @@ pub use physical_scan::PhysicalScan; pub use project::ProjectItem; pub use project::ProjectPlan; pub use scalar::*; +pub use sort::SortPlan; use crate::sql::optimizer::PhysicalProperty; use crate::sql::optimizer::RelationalProperty; @@ -82,6 +84,7 @@ pub enum PlanType { Project, Filter, Aggregate, + Sort, // Pattern Pattern, @@ -99,6 +102,7 @@ pub enum BasePlanImpl { Project(ProjectPlan), Filter(FilterPlan), Aggregate(AggregatePlan), + Sort(SortPlan), Pattern(PatternPlan), } diff --git a/query/src/sql/planner/plans/scalar.rs b/query/src/sql/planner/plans/scalar.rs index eba675b0e443..d3cfedb351cf 100644 --- a/query/src/sql/planner/plans/scalar.rs +++ b/query/src/sql/planner/plans/scalar.rs @@ -49,6 +49,7 @@ pub enum Scalar { AggregateFunction(AggregateFunction), FunctionCall(FunctionCall), Cast(CastExpr), + Order(OrderExpr), } #[derive(Clone, PartialEq, Debug)] @@ -249,3 +250,22 @@ impl ScalarExpr for CastExpr { self.argument.used_columns() } } + +#[derive(Clone, PartialEq, Debug)] +pub struct OrderExpr { + pub expr: Box, + // Optional `ASC` or `DESC` + pub asc: Option, + // Optional `NULLS FIRST` or `NULLS LAST` + pub nulls_first: Option, +} + +impl ScalarExpr for OrderExpr { + fn data_type(&self) -> DataTypeImpl { + todo!() + } + + fn used_columns(&self) -> ColumnSet { + self.expr.used_columns() + } +} diff --git a/query/src/sql/planner/plans/sort.rs b/query/src/sql/planner/plans/sort.rs new file mode 100644 index 000000000000..1e744cd2f98d --- /dev/null +++ b/query/src/sql/planner/plans/sort.rs @@ -0,0 +1,67 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::any::Any; + +use crate::sql::optimizer::PhysicalProperty; +use crate::sql::optimizer::RelationalProperty; +use crate::sql::optimizer::SExpr; +use crate::sql::plans::BasePlan; +use crate::sql::plans::LogicalPlan; +use crate::sql::plans::PhysicalPlan; +use crate::sql::plans::PlanType; +use crate::sql::plans::Scalar; + +#[derive(Clone, Debug)] +pub struct SortPlan { + pub items: Vec, +} + +impl BasePlan for SortPlan { + fn plan_type(&self) -> PlanType { + PlanType::Sort + } + + fn is_physical(&self) -> bool { + true + } + + fn is_logical(&self) -> bool { + true + } + + fn as_physical(&self) -> Option<&dyn PhysicalPlan> { + todo!() + } + + fn as_logical(&self) -> Option<&dyn LogicalPlan> { + todo!() + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +impl PhysicalPlan for SortPlan { + fn compute_physical_prop(&self, _expression: &SExpr) -> PhysicalProperty { + todo!() + } +} + +impl LogicalPlan for SortPlan { + fn compute_relational_prop(&self, _expression: &SExpr) -> RelationalProperty { + todo!() + } +} diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result index 8e858452484b..0954e388695c 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result @@ -72,7 +72,43 @@ 1 2 2 3 1000 +====ORDER_BY==== +2 0 +2 0 +2 1 +1 0 +1 1 +1 1 +0 0 +0 0 +0 1 +0 1 +2 NULL +1 NULL +0 NULL +0 1 +0 0 +0 1 +0 0 +1 1 +1 0 +1 1 +2 0 +2 1 +2 0 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +1 +2 ====SELECT_WITHOUT_FROM==== 2 8 -new_planner +new_planner \ No newline at end of file diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql index 5a3796ca26c7..ecb1de69b4c1 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql @@ -68,13 +68,21 @@ select * from t1 inner join t on t.a = t1.b; select * from t2 inner join t on t.a = t2.c; select * from t2 inner join t on t.a = t2.c + 1; select * from t2 inner join t on t.a = t2.c + 1 and t.a - 1 = t2.c; +select count(*) from numbers(1000) as t inner join numbers(1000) as t1 on t.number = t1.number; +-- order by +select '====ORDER_BY===='; +SELECT number%3 as c1, number%2 as c2 FROM numbers_mt (10) order by c1 desc, c2 asc; +SELECT number, null from numbers(3) order by number desc; +SELECT number%3 as c1, number%2 as c2 FROM numbers_mt (10) order by c1, number desc; +SELECT SUM(number) AS s FROM numbers_mt(10) GROUP BY number ORDER BY s +create table t3(a int, b int); +insert into t3 values(1,2),(2,3); +select a from t order by a + b; drop table t; drop table t1; drop table t2; -select count(*) from numbers(1000) as t inner join numbers(1000) as t1 on t.number = t1.number; - -- Select without from select '====SELECT_WITHOUT_FROM===='; select 1 + 1; From 43197fbef4e602164d3112f6e1a6e3cbfffa6aed Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 9 May 2022 15:20:07 +0800 Subject: [PATCH 02/14] fix conflicts --- tests/suites/0_stateless/20+_others/20_0001_planner_v2.result | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result index 0954e388695c..8dffd00c7b58 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result @@ -111,4 +111,5 @@ ====SELECT_WITHOUT_FROM==== 2 8 -new_planner \ No newline at end of file +new_planner + From 21c8caec5f0605df264f139acfb97667476bfabe Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 9 May 2022 15:53:59 +0800 Subject: [PATCH 03/14] refine comments and variable name --- query/src/sql/planner/binder/bind_context.rs | 7 ++++--- query/src/sql/planner/binder/project.rs | 4 ++-- query/src/sql/planner/binder/select.rs | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/query/src/sql/planner/binder/bind_context.rs b/query/src/sql/planner/binder/bind_context.rs index 89fa8161b6d9..4dfd85194d9b 100644 --- a/query/src/sql/planner/binder/bind_context.rs +++ b/query/src/sql/planner/binder/bind_context.rs @@ -51,10 +51,11 @@ pub struct BindContext { /// Aggregation scalar expression pub agg_scalar_exprs: Option>, - /// Order by columnBinding, consider the sql: select a as c from t1 order by c, a, + /// Order by columnBinding, consider the sql: select sum(a) from t group by a,b order by b;, /// Order by requires not just the columns in the selection, /// but the columns of the entire table as well as the columns of the selection - pub group_by_columns: Vec, + /// + pub order_by_columns: Vec, } impl BindContext { @@ -68,7 +69,7 @@ impl BindContext { columns: vec![], expression: None, agg_scalar_exprs: None, - group_by_columns: vec![], + order_by_columns: vec![], } } diff --git a/query/src/sql/planner/binder/project.rs b/query/src/sql/planner/binder/project.rs index 21d5797e2de0..458ee14bc3d4 100644 --- a/query/src/sql/planner/binder/project.rs +++ b/query/src/sql/planner/binder/project.rs @@ -76,7 +76,7 @@ impl Binder { input_context: &BindContext, ) -> Result { let mut output_context = BindContext::new(); - output_context.group_by_columns = input_context.columns.clone(); + output_context.order_by_columns = input_context.columns.clone(); output_context.expression = input_context.expression.clone(); for select_target in select_list { match select_target { @@ -127,7 +127,7 @@ impl Binder { scalar: Some(Box::new(bound_expr.clone())), }; if !matches!(bound_expr, Scalar::BoundColumnRef(BoundColumnRef { .. })) { - output_context.group_by_columns.push(column_binding.clone()); + output_context.order_by_columns.push(column_binding.clone()); } output_context.add_column_binding(column_binding); } diff --git a/query/src/sql/planner/binder/select.rs b/query/src/sql/planner/binder/select.rs index 2093d5d24b02..c492fe08b759 100644 --- a/query/src/sql/planner/binder/select.rs +++ b/query/src/sql/planner/binder/select.rs @@ -57,7 +57,7 @@ impl Binder { if !query.order_by.is_empty() { let bind_context_cols = bind_context.columns.clone(); - bind_context.columns = bind_context.group_by_columns.clone(); + bind_context.columns = bind_context.order_by_columns.clone(); self.bind_order_by(&query.order_by, &mut bind_context)?; bind_context.columns = bind_context_cols; } From a154f1b2640e547de79986a6d380dd428da83812 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 9 May 2022 17:41:42 +0800 Subject: [PATCH 04/14] address commments --- query/src/sql/planner/binder/bind_context.rs | 5 ++--- query/src/sql/planner/binder/project.rs | 15 ++++++++++++--- query/src/sql/planner/binder/select.rs | 19 ++++++++++++++++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/query/src/sql/planner/binder/bind_context.rs b/query/src/sql/planner/binder/bind_context.rs index 4dfd85194d9b..fe210da948a8 100644 --- a/query/src/sql/planner/binder/bind_context.rs +++ b/query/src/sql/planner/binder/bind_context.rs @@ -54,8 +54,7 @@ pub struct BindContext { /// Order by columnBinding, consider the sql: select sum(a) from t group by a,b order by b;, /// Order by requires not just the columns in the selection, /// but the columns of the entire table as well as the columns of the selection - /// - pub order_by_columns: Vec, + pub order_by_columns: Option>, } impl BindContext { @@ -69,7 +68,7 @@ impl BindContext { columns: vec![], expression: None, agg_scalar_exprs: None, - order_by_columns: vec![], + order_by_columns: Some(Vec::new()), } } diff --git a/query/src/sql/planner/binder/project.rs b/query/src/sql/planner/binder/project.rs index 458ee14bc3d4..39562a620740 100644 --- a/query/src/sql/planner/binder/project.rs +++ b/query/src/sql/planner/binder/project.rs @@ -73,10 +73,13 @@ impl Binder { pub(super) fn normalize_select_list( &mut self, select_list: &[SelectTarget], + has_order_by: bool, input_context: &BindContext, ) -> Result { let mut output_context = BindContext::new(); - output_context.order_by_columns = input_context.columns.clone(); + if has_order_by { + output_context.order_by_columns = Some(input_context.columns.clone()); + } output_context.expression = input_context.expression.clone(); for select_target in select_list { match select_target { @@ -126,8 +129,14 @@ impl Binder { data_type, scalar: Some(Box::new(bound_expr.clone())), }; - if !matches!(bound_expr, Scalar::BoundColumnRef(BoundColumnRef { .. })) { - output_context.order_by_columns.push(column_binding.clone()); + if has_order_by + && !matches!(bound_expr, Scalar::BoundColumnRef(BoundColumnRef { .. })) + { + output_context + .order_by_columns + .as_mut() + .unwrap() + .push(column_binding.clone()); } output_context.add_column_binding(column_binding); } diff --git a/query/src/sql/planner/binder/select.rs b/query/src/sql/planner/binder/select.rs index c492fe08b759..c6fc1b83d2eb 100644 --- a/query/src/sql/planner/binder/select.rs +++ b/query/src/sql/planner/binder/select.rs @@ -49,15 +49,26 @@ impl Binder { query: &Query, bind_context: &BindContext, ) -> Result { + let mut has_order_by = false; + if !query.order_by.is_empty() { + has_order_by = true; + }; let mut bind_context = match &query.body { - SetExpr::Select(stmt) => self.bind_select_stmt(stmt, bind_context).await, + SetExpr::Select(stmt) => { + self.bind_select_stmt(stmt, has_order_by, bind_context) + .await + } SetExpr::Query(stmt) => self.bind_query(stmt, bind_context).await, _ => Err(ErrorCode::UnImplement("Unsupported query type")), }?; if !query.order_by.is_empty() { let bind_context_cols = bind_context.columns.clone(); - bind_context.columns = bind_context.order_by_columns.clone(); + bind_context.columns = bind_context + .order_by_columns + .as_ref() + .ok_or_else(|| ErrorCode::LogicalError("Order by should have order by columns"))? + .clone(); self.bind_order_by(&query.order_by, &mut bind_context)?; bind_context.columns = bind_context_cols; } @@ -72,6 +83,7 @@ impl Binder { pub(super) async fn bind_select_stmt( &mut self, stmt: &SelectStmt, + has_order_by: bool, bind_context: &BindContext, ) -> Result { let mut input_context = if let Some(from) = &stmt.from { @@ -85,7 +97,8 @@ impl Binder { } // Output of current `SELECT` statement. - let mut output_context = self.normalize_select_list(&stmt.select_list, &input_context)?; + let mut output_context = + self.normalize_select_list(&stmt.select_list, has_order_by, &input_context)?; self.analyze_aggregate(&output_context, &mut input_context)?; From 126c46464bdfd037cd588e430131c13676520012 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 9 May 2022 21:52:19 +0800 Subject: [PATCH 05/14] order by support non-column expression except agg expr --- query/src/sql/exec/data_schema_builder.rs | 20 ++++++++++++++++ query/src/sql/exec/mod.rs | 23 +++++++++++++++---- query/src/sql/planner/binder/project.rs | 4 +++- query/src/sql/planner/binder/select.rs | 4 ++-- .../20+_others/20_0001_planner_v2.sql | 1 + 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/query/src/sql/exec/data_schema_builder.rs b/query/src/sql/exec/data_schema_builder.rs index b0a5fca5d0e0..c0af8f2ce39d 100644 --- a/query/src/sql/exec/data_schema_builder.rs +++ b/query/src/sql/exec/data_schema_builder.rs @@ -191,4 +191,24 @@ impl<'a> DataSchemaBuilder<'a> { DataSchemaRefExt::create(fields) } + + pub fn build_sort( + &self, + input_schema: &DataSchemaRef, + exprs: &[Expression], + ) -> Result { + let mut fields = input_schema.fields().clone(); + for expr in exprs.iter() { + let expr_name = expr.column_name().clone(); + if !input_schema.has_field(expr_name.as_str()) { + let field = if expr.nullable(&input_schema)? { + DataField::new_nullable(expr_name.as_str(), expr.to_data_type(&input_schema)?) + } else { + DataField::new(expr_name.as_str(), expr.to_data_type(&input_schema)?) + }; + fields.push(field); + } + } + Ok(DataSchemaRefExt::create(fields)) + } } diff --git a/query/src/sql/exec/mod.rs b/query/src/sql/exec/mod.rs index c2414ad7b782..8c28746b7db3 100644 --- a/query/src/sql/exec/mod.rs +++ b/query/src/sql/exec/mod.rs @@ -500,13 +500,26 @@ impl PipelineBuilder { input_schema: DataSchemaRef, pipeline: &mut NewPipeline, ) -> Result { - let output_schema = input_schema.clone(); let eb = ExpressionBuilder::create(&self.metadata); let mut expressions = Vec::with_capacity(sort_plan.items.len()); for item in sort_plan.items.iter() { expressions.push(eb.build(item)?); } + let schema_builder = DataSchemaBuilder::new(&self.metadata); + let output_schema = schema_builder.build_sort(&input_schema, expressions.as_slice())?; + + pipeline.add_transform(|transform_input_port, transform_output_port| { + ExpressionTransform::try_create( + transform_input_port, + transform_output_port, + input_schema.clone(), + output_schema.clone(), + expressions.clone(), + self.ctx.clone(), + ) + })?; + //TODO(xudong963): Add rows_limit // processor 1: block ---> sort_stream @@ -517,7 +530,7 @@ impl PipelineBuilder { transform_input_port, transform_output_port, None, - get_sort_descriptions(&input_schema, expressions.as_slice())?, + get_sort_descriptions(&output_schema, expressions.as_slice())?, ) })?; @@ -530,7 +543,7 @@ impl PipelineBuilder { transform_output_port, SortMergeCompactor::new( None, - get_sort_descriptions(&input_schema, expressions.as_slice())?, + get_sort_descriptions(&output_schema, expressions.as_slice())?, ), ) })?; @@ -547,11 +560,11 @@ impl PipelineBuilder { transform_output_port, SortMergeCompactor::new( None, - get_sort_descriptions(&input_schema, expressions.as_slice())?, + get_sort_descriptions(&output_schema, expressions.as_slice())?, ), ) })?; - Ok(output_schema) + Ok(output_schema.clone()) } } diff --git a/query/src/sql/planner/binder/project.rs b/query/src/sql/planner/binder/project.rs index 39562a620740..8727499cb8b8 100644 --- a/query/src/sql/planner/binder/project.rs +++ b/query/src/sql/planner/binder/project.rs @@ -135,7 +135,9 @@ impl Binder { output_context .order_by_columns .as_mut() - .unwrap() + .ok_or_else(|| { + ErrorCode::SemanticError("Order by should have order by columns") + })? .push(column_binding.clone()); } output_context.add_column_binding(column_binding); diff --git a/query/src/sql/planner/binder/select.rs b/query/src/sql/planner/binder/select.rs index c6fc1b83d2eb..ec51c3a75f57 100644 --- a/query/src/sql/planner/binder/select.rs +++ b/query/src/sql/planner/binder/select.rs @@ -62,12 +62,12 @@ impl Binder { _ => Err(ErrorCode::UnImplement("Unsupported query type")), }?; - if !query.order_by.is_empty() { + if has_order_by { let bind_context_cols = bind_context.columns.clone(); bind_context.columns = bind_context .order_by_columns .as_ref() - .ok_or_else(|| ErrorCode::LogicalError("Order by should have order by columns"))? + .ok_or_else(|| ErrorCode::SemanticError("Order by should have order by columns"))? .clone(); self.bind_order_by(&query.order_by, &mut bind_context)?; bind_context.columns = bind_context_cols; diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql index ecb1de69b4c1..1b7bfdb2ebc3 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql @@ -82,6 +82,7 @@ select a from t order by a + b; drop table t; drop table t1; drop table t2; +drop table t3; -- Select without from select '====SELECT_WITHOUT_FROM===='; From 8733b6331c907a60a397e5831d9ca82124c80373 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 9 May 2022 22:10:06 +0800 Subject: [PATCH 06/14] fix conflicts --- query/src/sql/exec/data_schema_builder.rs | 6 +++--- .../suites/0_stateless/20+_others/20_0001_planner_v2.result | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/query/src/sql/exec/data_schema_builder.rs b/query/src/sql/exec/data_schema_builder.rs index c0af8f2ce39d..7f63e71dbf99 100644 --- a/query/src/sql/exec/data_schema_builder.rs +++ b/query/src/sql/exec/data_schema_builder.rs @@ -201,10 +201,10 @@ impl<'a> DataSchemaBuilder<'a> { for expr in exprs.iter() { let expr_name = expr.column_name().clone(); if !input_schema.has_field(expr_name.as_str()) { - let field = if expr.nullable(&input_schema)? { - DataField::new_nullable(expr_name.as_str(), expr.to_data_type(&input_schema)?) + let field = if expr.nullable(input_schema)? { + DataField::new_nullable(expr_name.as_str(), expr.to_data_type(input_schema)?) } else { - DataField::new(expr_name.as_str(), expr.to_data_type(&input_schema)?) + DataField::new(expr_name.as_str(), expr.to_data_type(input_schema)?) }; fields.push(field); } diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result index 8dffd00c7b58..0954e388695c 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result @@ -111,5 +111,4 @@ ====SELECT_WITHOUT_FROM==== 2 8 -new_planner - +new_planner \ No newline at end of file From b703f594d4b1449b61faae41701e86eeab077975 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 9 May 2022 22:18:28 +0800 Subject: [PATCH 07/14] fix test --- tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql index 1b7bfdb2ebc3..2da07e42996a 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql @@ -78,7 +78,7 @@ SELECT number%3 as c1, number%2 as c2 FROM numbers_mt (10) order by c1, number d SELECT SUM(number) AS s FROM numbers_mt(10) GROUP BY number ORDER BY s create table t3(a int, b int); insert into t3 values(1,2),(2,3); -select a from t order by a + b; +select a from t3 order by a + b; drop table t; drop table t1; drop table t2; From fc991414e4caeb55d81c6c5540f628cf748d1652 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 9 May 2022 22:50:52 +0800 Subject: [PATCH 08/14] fix test --- tests/suites/0_stateless/20+_others/20_0001_planner_v2.result | 2 +- tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result index 0954e388695c..4e83c8fd60b4 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.result @@ -111,4 +111,4 @@ ====SELECT_WITHOUT_FROM==== 2 8 -new_planner \ No newline at end of file +new_planner diff --git a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql index 2da07e42996a..e1254a69b962 100644 --- a/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql +++ b/tests/suites/0_stateless/20+_others/20_0001_planner_v2.sql @@ -75,7 +75,7 @@ select '====ORDER_BY===='; SELECT number%3 as c1, number%2 as c2 FROM numbers_mt (10) order by c1 desc, c2 asc; SELECT number, null from numbers(3) order by number desc; SELECT number%3 as c1, number%2 as c2 FROM numbers_mt (10) order by c1, number desc; -SELECT SUM(number) AS s FROM numbers_mt(10) GROUP BY number ORDER BY s +SELECT SUM(number) AS s FROM numbers_mt(10) GROUP BY number ORDER BY s; create table t3(a int, b int); insert into t3 values(1,2),(2,3); select a from t3 order by a + b; From fae4f0fabf9eac58ae28a80792a79f7e32773dd8 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Tue, 10 May 2022 11:34:52 +0800 Subject: [PATCH 09/14] chore(planner): make code neat with --- query/src/sql/exec/data_schema_builder.rs | 49 +++++------------------ 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/query/src/sql/exec/data_schema_builder.rs b/query/src/sql/exec/data_schema_builder.rs index 7f63e71dbf99..721bcde688f4 100644 --- a/query/src/sql/exec/data_schema_builder.rs +++ b/query/src/sql/exec/data_schema_builder.rs @@ -15,7 +15,6 @@ use common_datavalues::DataField; use common_datavalues::DataSchemaRef; use common_datavalues::DataSchemaRefExt; -use common_datavalues::DataTypeImpl; use common_exception::ErrorCode; use common_exception::Result; use common_planners::Expression; @@ -46,11 +45,7 @@ impl<'a> DataSchemaBuilder<'a> { new_data_fields.push(data_field); continue; } - let field = if data_field.is_nullable() { - DataField::new_nullable(data_field.name(), data_field.data_type().clone()) - } else { - DataField::new(data_field.name(), data_field.data_type().clone()) - }; + let field = DataField::new(data_field.name(), data_field.data_type().clone()); new_data_fields.push(field); } Ok(DataSchemaRefExt::create(new_data_fields)) @@ -78,12 +73,8 @@ impl<'a> DataSchemaBuilder<'a> { for index in plan.columns.iter() { let column_entry = self.metadata.column(*index); let field_name = format_field_name(column_entry.name.as_str(), *index); - let field = if matches!(column_entry.data_type, DataTypeImpl::Nullable(_)) { - DataField::new_nullable(field_name.as_str(), column_entry.data_type.clone()) - } else { - DataField::new(field_name.as_str(), column_entry.data_type.clone()) - }; - + let field = DataField::new(field_name.as_str(), column_entry.data_type.clone()); + dbg!(field.clone()); fields.push(field); } @@ -95,11 +86,8 @@ impl<'a> DataSchemaBuilder<'a> { for index in columns { let column_entry = self.metadata.column(*index); let field_name = column_entry.name.clone(); - let field = if matches!(column_entry.data_type, DataTypeImpl::Nullable(_)) { - DataField::new_nullable(field_name.as_str(), column_entry.data_type.clone()) - } else { - DataField::new(field_name.as_str(), column_entry.data_type.clone()) - }; + let field = + DataField::new_nullable(field_name.as_str(), column_entry.data_type.clone()); fields.push(field); } @@ -124,11 +112,7 @@ impl<'a> DataSchemaBuilder<'a> { if input_schema.has_field(expr_name.as_str()) { continue; } - let field = if expr.nullable(&input_schema)? { - DataField::new_nullable(expr_name.as_str(), expr.to_data_type(&input_schema)?) - } else { - DataField::new(expr_name.as_str(), expr.to_data_type(&input_schema)?) - }; + let field = DataField::new(expr_name.as_str(), expr.to_data_type(&input_schema)?); fields.push(field); } Ok(DataSchemaRefExt::create(fields)) @@ -154,17 +138,10 @@ impl<'a> DataSchemaBuilder<'a> { if input_schema.has_field(expr_name.as_str()) { continue; } - let field = if arg_inner_expr.nullable(&input_schema)? { - DataField::new_nullable( - expr_name.as_str(), - arg_inner_expr.to_data_type(&input_schema)?, - ) - } else { - DataField::new( - expr_name.as_str(), - arg_inner_expr.to_data_type(&input_schema)?, - ) - }; + let field = DataField::new( + expr_name.as_str(), + arg_inner_expr.to_data_type(&input_schema)?, + ); fields.push(field); agg_inner_expressions.push(arg_inner_expr.clone()) } @@ -201,11 +178,7 @@ impl<'a> DataSchemaBuilder<'a> { for expr in exprs.iter() { let expr_name = expr.column_name().clone(); if !input_schema.has_field(expr_name.as_str()) { - let field = if expr.nullable(input_schema)? { - DataField::new_nullable(expr_name.as_str(), expr.to_data_type(input_schema)?) - } else { - DataField::new(expr_name.as_str(), expr.to_data_type(input_schema)?) - }; + let field = DataField::new(expr_name.as_str(), expr.to_data_type(input_schema)?); fields.push(field); } } From 0500b9ffe344594fda80f1bfb2f878f4e703123a Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 10 May 2022 11:40:32 +0800 Subject: [PATCH 10/14] refactor: Move configs to common so we can reuse it Signed-off-by: Xuanwo --- Cargo.lock | 21 +++++++++++++- Cargo.toml | 1 + common/configs/Cargo.toml | 29 +++++++++++++++++++ .../configs => common/configs/src}/config.rs | 16 +++------- .../mod.rs => common/configs/src/lib.rs | 22 +++++++------- .../configs/src/log.rs | 0 .../configs/src/meta.rs | 29 ------------------- .../configs/src/query.rs | 0 .../configs/src/storage.rs | 0 .../it => common/configs/tests}/configs.rs | 4 +-- common/configs/tests/main.rs | 14 +++++++++ common/grpc/Cargo.toml | 1 + common/grpc/src/client_conf.rs | 21 ++++++++++++++ common/meta/grpc/Cargo.toml | 1 + common/meta/grpc/src/grpc_client_conf.rs | 23 +++++++++++++++ query/Cargo.toml | 2 +- query/benches/suites/mod.rs | 2 +- query/src/api/http_service.rs | 2 +- query/src/api/rpc_service.rs | 2 +- query/src/bin/databend-query.rs | 7 ++--- query/src/catalogs/impls/database_catalog.rs | 2 +- query/src/catalogs/impls/immutable_catalog.rs | 2 +- query/src/catalogs/impls/mutable_catalog.rs | 8 +++-- query/src/clusters/cluster.rs | 8 +++-- query/src/databases/database_factory.rs | 2 +- query/src/databases/system/system_database.rs | 2 +- query/src/lib.rs | 1 - query/src/servers/http/http_services.rs | 2 +- .../http/v1/query/http_query_manager.rs | 2 +- query/src/servers/mysql/mysql_federated.rs | 2 +- .../servers/mysql/mysql_interactive_worker.rs | 2 +- query/src/sessions/query_ctx.rs | 4 +-- query/src/sessions/query_ctx_shared.rs | 2 +- query/src/sessions/session.rs | 2 +- query/src/sessions/session_ctx.rs | 2 +- query/src/sessions/session_mgr.rs | 2 +- query/src/sessions/session_settings.rs | 2 +- query/src/storages/cache.rs | 3 +- query/src/storages/storage_factory.rs | 2 +- query/src/users/auth/auth_mgr.rs | 2 +- query/src/users/auth/jwt/authenticator.rs | 3 +- query/src/users/user_api.rs | 5 ++-- query/tests/it/main.rs | 1 - query/tests/it/sessions/query_ctx.rs | 4 +-- query/tests/it/sessions/session_context.rs | 2 +- query/tests/it/tests/config.rs | 2 +- query/tests/it/tests/context.rs | 2 +- query/tests/it/tests/sessions.rs | 2 +- 48 files changed, 172 insertions(+), 100 deletions(-) create mode 100644 common/configs/Cargo.toml rename {query/src/configs => common/configs/src}/config.rs (88%) rename query/src/configs/mod.rs => common/configs/src/lib.rs (65%) rename query/src/configs/config_log.rs => common/configs/src/log.rs (100%) rename query/src/configs/config_meta.rs => common/configs/src/meta.rs (78%) rename query/src/configs/config_query.rs => common/configs/src/query.rs (100%) rename query/src/configs/config_storage.rs => common/configs/src/storage.rs (100%) rename {query/tests/it => common/configs/tests}/configs.rs (98%) create mode 100644 common/configs/tests/main.rs diff --git a/Cargo.lock b/Cargo.lock index 991ee516097c..4be37f2f57f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -903,6 +903,23 @@ dependencies = [ "walkdir", ] +[[package]] +name = "common-configs" +version = "0.1.0" +dependencies = [ + "clap 3.1.8", + "common-base", + "common-exception", + "once_cell", + "pretty_assertions", + "serde", + "serde-bridge", + "serde_json", + "serfig", + "temp-env", + "toml", +] + [[package]] name = "common-contexts" version = "0.1.0" @@ -1023,6 +1040,7 @@ version = "0.1.0" dependencies = [ "anyerror", "common-base", + "common-configs", "common-exception", "common-tracing", "hyper", @@ -1122,6 +1140,7 @@ version = "0.1.0" dependencies = [ "common-arrow", "common-base", + "common-configs", "common-exception", "common-grpc", "common-meta-api", @@ -1763,6 +1782,7 @@ dependencies = [ "common-base", "common-building", "common-cache", + "common-configs", "common-contexts", "common-datablocks", "common-datavalues", @@ -1821,7 +1841,6 @@ dependencies = [ "sha1", "sha2 0.10.2", "sqlparser", - "temp-env", "tempfile", "thiserror", "threadpool", diff --git a/Cargo.toml b/Cargo.toml index 8bedfc7e9a76..f38803883577 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "common/base", "common/building", "common/cache", + "common/configs", "common/contexts", "common/datablocks", "common/datavalues", diff --git a/common/configs/Cargo.toml b/common/configs/Cargo.toml new file mode 100644 index 000000000000..39d397c3c7d8 --- /dev/null +++ b/common/configs/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "common-configs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +doctest = false +test = false + +[dependencies] +# Workspace dependencies +common-base = { path = "../../common/base" } +common-exception = {path = "../../common/exception"} + +# Github dependencies + +# Crates.io dependencies +clap = { version = "3.1.8", features = ["derive", "env"] } +serde = { version = "1.0.136", features = ["derive"] } +serde-bridge = "0.0.3" +serde_json = "1.0.79" +serfig = "0.0.1" +once_cell = "1.10.0" + +[dev-dependencies] +pretty_assertions = "1.2.1" +toml = { version = "0.5.8", default-features = false } +temp-env = "0.2.0" diff --git a/query/src/configs/config.rs b/common/configs/src/config.rs similarity index 88% rename from query/src/configs/config.rs rename to common/configs/src/config.rs index 79dc3401ce31..52b4c8251189 100644 --- a/query/src/configs/config.rs +++ b/common/configs/src/config.rs @@ -16,7 +16,6 @@ use std::env; use clap::Parser; use common_exception::Result; -use common_grpc::RpcClientTlsConfig; use once_cell::sync::Lazy; use serde::Deserialize; use serde::Serialize; @@ -25,10 +24,10 @@ use serfig::collectors::from_file; use serfig::collectors::from_self; use serfig::parsers::Toml; -use crate::configs::LogConfig; -use crate::configs::MetaConfig; -use crate::configs::QueryConfig; -use crate::configs::StorageConfig; +use crate::LogConfig; +use crate::MetaConfig; +use crate::QueryConfig; +use crate::StorageConfig; pub static DATABEND_COMMIT_VERSION: Lazy = Lazy::new(|| { let git_tag = option_env!("VERGEN_GIT_SEMVER"); @@ -104,13 +103,6 @@ impl Config { Ok(builder.build()?) } - pub fn tls_query_client_conf(&self) -> RpcClientTlsConfig { - RpcClientTlsConfig { - rpc_tls_server_root_ca_cert: self.query.rpc_tls_query_server_root_ca_cert.to_string(), - domain_name: self.query.rpc_tls_query_service_domain_name.to_string(), - } - } - pub fn tls_query_cli_enabled(&self) -> bool { !self.query.rpc_tls_query_server_root_ca_cert.is_empty() && !self.query.rpc_tls_query_service_domain_name.is_empty() diff --git a/query/src/configs/mod.rs b/common/configs/src/lib.rs similarity index 65% rename from query/src/configs/mod.rs rename to common/configs/src/lib.rs index 2fbe0b8eec03..3529bd155a8d 100644 --- a/query/src/configs/mod.rs +++ b/common/configs/src/lib.rs @@ -13,17 +13,17 @@ // limitations under the License. mod config; -pub mod config_log; -pub mod config_meta; -pub mod config_query; -pub mod config_storage; +pub mod log; +pub mod meta; +pub mod query; +pub mod storage; pub use config::Config; pub use config::DATABEND_COMMIT_VERSION; -pub use config_log::LogConfig; -pub use config_meta::MetaConfig; -pub use config_query::QueryConfig; -pub use config_storage::AzblobStorageConfig; -pub use config_storage::FsStorageConfig; -pub use config_storage::S3StorageConfig; -pub use config_storage::StorageConfig; +pub use log::LogConfig; +pub use meta::MetaConfig; +pub use query::QueryConfig; +pub use storage::AzblobStorageConfig; +pub use storage::FsStorageConfig; +pub use storage::S3StorageConfig; +pub use storage::StorageConfig; diff --git a/query/src/configs/config_log.rs b/common/configs/src/log.rs similarity index 100% rename from query/src/configs/config_log.rs rename to common/configs/src/log.rs diff --git a/query/src/configs/config_meta.rs b/common/configs/src/meta.rs similarity index 78% rename from query/src/configs/config_meta.rs rename to common/configs/src/meta.rs index d5db47f8fd07..a2ead218cf6a 100644 --- a/query/src/configs/config_meta.rs +++ b/common/configs/src/meta.rs @@ -16,9 +16,6 @@ use std::fmt; use clap::Args; use common_base::base::mask_string; -use common_grpc::RpcClientConf; -use common_grpc::RpcClientTlsConfig; -use common_meta_grpc::MetaGrpcClientConf; use serde::Deserialize; use serde::Serialize; @@ -79,32 +76,6 @@ impl MetaConfig { !self.rpc_tls_meta_server_root_ca_cert.is_empty() && !self.rpc_tls_meta_service_domain_name.is_empty() } - - pub fn to_grpc_tls_config(&self) -> Option { - if !self.is_tls_enabled() { - return None; - } - - Some(RpcClientTlsConfig { - rpc_tls_server_root_ca_cert: self.rpc_tls_meta_server_root_ca_cert.clone(), - domain_name: self.rpc_tls_meta_service_domain_name.clone(), - }) - } - - pub fn to_grpc_client_config(&self) -> MetaGrpcClientConf { - let meta_config = RpcClientConf { - address: self.address.clone(), - username: self.username.clone(), - password: self.password.clone(), - tls_conf: self.to_grpc_tls_config(), - }; - - MetaGrpcClientConf { - meta_service_config: meta_config.clone(), - kv_service_config: meta_config, - client_timeout_in_second: self.client_timeout_in_second, - } - } } impl fmt::Debug for MetaConfig { diff --git a/query/src/configs/config_query.rs b/common/configs/src/query.rs similarity index 100% rename from query/src/configs/config_query.rs rename to common/configs/src/query.rs diff --git a/query/src/configs/config_storage.rs b/common/configs/src/storage.rs similarity index 100% rename from query/src/configs/config_storage.rs rename to common/configs/src/storage.rs diff --git a/query/tests/it/configs.rs b/common/configs/tests/configs.rs similarity index 98% rename from query/tests/it/configs.rs rename to common/configs/tests/configs.rs index fd111d0defc9..69d53494bd6f 100644 --- a/query/tests/it/configs.rs +++ b/common/configs/tests/configs.rs @@ -16,8 +16,8 @@ use std::env::temp_dir; use std::fs; use std::io::Write; +use common_configs::Config; use common_exception::Result; -use databend_query::configs::Config; use pretty_assertions::assert_eq; // Default. @@ -303,7 +303,7 @@ root = "" #[test] fn test_fuse_commit_version() -> Result<()> { - let v = &databend_query::configs::DATABEND_COMMIT_VERSION; + let v = &common_configs::DATABEND_COMMIT_VERSION; assert!(v.len() > 0); Ok(()) } diff --git a/common/configs/tests/main.rs b/common/configs/tests/main.rs new file mode 100644 index 000000000000..de888d3a63e7 --- /dev/null +++ b/common/configs/tests/main.rs @@ -0,0 +1,14 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +mod configs; diff --git a/common/grpc/Cargo.toml b/common/grpc/Cargo.toml index 607cf678145e..bbb2e19ef602 100644 --- a/common/grpc/Cargo.toml +++ b/common/grpc/Cargo.toml @@ -15,6 +15,7 @@ test = false common-base = { path = "../base" } common-exception = { path = "../exception" } common-tracing = { path = "../tracing" } +common-configs = {path = "../configs"} # Github dependencies diff --git a/common/grpc/src/client_conf.rs b/common/grpc/src/client_conf.rs index d21f16ec4372..75b30ec24c89 100644 --- a/common/grpc/src/client_conf.rs +++ b/common/grpc/src/client_conf.rs @@ -13,6 +13,9 @@ // limitations under the License. // +use common_configs::MetaConfig; +use common_configs::QueryConfig; + #[derive(Clone, Debug, Default)] pub struct RpcClientTlsConfig { pub rpc_tls_server_root_ca_cert: String, @@ -25,6 +28,24 @@ impl RpcClientTlsConfig { } } +impl From<&QueryConfig> for RpcClientTlsConfig { + fn from(qc: &QueryConfig) -> Self { + RpcClientTlsConfig { + rpc_tls_server_root_ca_cert: qc.rpc_tls_query_server_root_ca_cert.to_string(), + domain_name: qc.rpc_tls_query_service_domain_name.to_string(), + } + } +} + +impl From<&MetaConfig> for RpcClientTlsConfig { + fn from(mc: &MetaConfig) -> Self { + RpcClientTlsConfig { + rpc_tls_server_root_ca_cert: mc.rpc_tls_meta_server_root_ca_cert.to_string(), + domain_name: mc.rpc_tls_meta_service_domain_name.to_string(), + } + } +} + #[derive(Clone, Debug, Default)] pub struct RpcClientConf { pub address: String, diff --git a/common/meta/grpc/Cargo.toml b/common/meta/grpc/Cargo.toml index 9c8dafb7dca3..00a3ae05d096 100644 --- a/common/meta/grpc/Cargo.toml +++ b/common/meta/grpc/Cargo.toml @@ -23,6 +23,7 @@ common-meta-types = { path = "../types" } common-proto-conv = { path = "../../proto-conv" } common-protos = { path = "../../protos" } common-tracing = { path = "../../tracing" } +common-configs = {path = "../../configs"} derive_more = "0.99.17" futures = "0.3.21" diff --git a/common/meta/grpc/src/grpc_client_conf.rs b/common/meta/grpc/src/grpc_client_conf.rs index b682c5d4d6a6..559e79c982ba 100644 --- a/common/meta/grpc/src/grpc_client_conf.rs +++ b/common/meta/grpc/src/grpc_client_conf.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use common_configs::MetaConfig; use common_grpc::RpcClientConf; +use common_grpc::RpcClientTlsConfig; #[derive(Clone, Debug, Default)] pub struct MetaGrpcClientConf { @@ -20,3 +22,24 @@ pub struct MetaGrpcClientConf { pub kv_service_config: RpcClientConf, pub client_timeout_in_second: u64, } + +impl From<&MetaConfig> for MetaGrpcClientConf { + fn from(mc: &MetaConfig) -> Self { + let meta_config = RpcClientConf { + address: mc.address.clone(), + username: mc.username.clone(), + password: mc.password.clone(), + tls_conf: if mc.is_tls_enabled() { + Some(RpcClientTlsConfig::from(mc)) + } else { + None + }, + }; + + MetaGrpcClientConf { + meta_service_config: meta_config.clone(), + kv_service_config: meta_config, + client_timeout_in_second: mc.client_timeout_in_second, + } + } +} diff --git a/query/Cargo.toml b/query/Cargo.toml index ee3285538856..a5ad42f516c6 100644 --- a/query/Cargo.toml +++ b/query/Cargo.toml @@ -49,6 +49,7 @@ common-metrics = { path = "../common/metrics" } common-planners = { path = "../common/planners" } common-streams = { path = "../common/streams" } common-tracing = { path = "../common/tracing" } +common-configs = {path ="../common/configs"} # Github dependencies bincode = { git = "https://github.com/datafuse-extras/bincode", rev = "fd3f9ff" } @@ -123,7 +124,6 @@ maplit = "1.0.2" mysql_async = "0.29.0" pretty_assertions = "1.2.1" reqwest = { version = "0.11.10", features = ["json", "native-tls"] } -temp-env = "0.2.0" tempfile = "3.3.0" toml = { version = "0.5.8", default-features = false } url = "2.2.2" diff --git a/query/benches/suites/mod.rs b/query/benches/suites/mod.rs index 20d99bff1743..32f07e786a84 100644 --- a/query/benches/suites/mod.rs +++ b/query/benches/suites/mod.rs @@ -13,10 +13,10 @@ // limitations under the License. use common_base::base::tokio; +use common_configs::Config; use common_exception::Result; use common_planners::PlanNode; use criterion::Criterion; -use databend_query::configs::Config; use databend_query::interpreters::SelectInterpreter; use databend_query::sessions::SessionManager; use databend_query::sessions::SessionType; diff --git a/query/src/api/http_service.rs b/query/src/api/http_service.rs index f9303aa9f512..cb6aa09ffc7d 100644 --- a/query/src/api/http_service.rs +++ b/query/src/api/http_service.rs @@ -16,6 +16,7 @@ use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; +use common_configs::Config; use common_exception::Result; use common_tracing::tracing; use poem::get; @@ -25,7 +26,6 @@ use poem::EndpointExt; use poem::Route; use crate::common::service::HttpShutdownHandler; -use crate::configs::Config; use crate::servers::Server; use crate::sessions::SessionManager; diff --git a/query/src/api/rpc_service.rs b/query/src/api/rpc_service.rs index 6c05d4d8075a..1778c6b6b53a 100644 --- a/query/src/api/rpc_service.rs +++ b/query/src/api/rpc_service.rs @@ -20,6 +20,7 @@ use common_arrow::arrow_format::flight::service::flight_service_server::FlightSe use common_base::base::tokio; use common_base::base::tokio::net::TcpListener; use common_base::base::tokio::sync::Notify; +use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_tracing::tracing; @@ -30,7 +31,6 @@ use tonic::transport::ServerTlsConfig; use crate::api::rpc::DatabendQueryFlightDispatcher; use crate::api::rpc::DatabendQueryFlightService; -use crate::configs::Config; use crate::servers::Server as DatabendQueryServer; use crate::sessions::SessionManager; diff --git a/query/src/bin/databend-query.rs b/query/src/bin/databend-query.rs index 9b47c0cb86c7..bca1a3c74846 100644 --- a/query/src/bin/databend-query.rs +++ b/query/src/bin/databend-query.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use common_base::base::RuntimeTracker; +use common_configs::Config; use common_macros::databend_main; use common_meta_embedded::MetaEmbedded; use common_metrics::init_default_metrics_recorder; @@ -23,7 +24,6 @@ use common_tracing::set_panic_hook; use common_tracing::tracing; use databend_query::api::HttpService; use databend_query::api::RpcService; -use databend_query::configs::Config; use databend_query::metrics::MetricService; use databend_query::servers::ClickHouseHandler; use databend_query::servers::HttpHandler; @@ -55,10 +55,7 @@ async fn main(_global_tracker: Arc) -> common_exception::Result< set_panic_hook(); tracing::info!("{:?}", conf); - tracing::info!( - "DatabendQuery {}", - *databend_query::configs::DATABEND_COMMIT_VERSION, - ); + tracing::info!("DatabendQuery {}", *common_configs::DATABEND_COMMIT_VERSION); let session_manager = SessionManager::from_conf(conf.clone()).await?; let mut shutdown_handle = ShutdownHandle::create(session_manager.clone()); diff --git a/query/src/catalogs/impls/database_catalog.rs b/query/src/catalogs/impls/database_catalog.rs index 7f16d0e29457..f4f61f90e1b1 100644 --- a/query/src/catalogs/impls/database_catalog.rs +++ b/query/src/catalogs/impls/database_catalog.rs @@ -15,6 +15,7 @@ use std::sync::Arc; +use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::CreateDatabaseReply; @@ -36,7 +37,6 @@ use common_tracing::tracing; use crate::catalogs::catalog::Catalog; use crate::catalogs::impls::ImmutableCatalog; use crate::catalogs::impls::MutableCatalog; -use crate::configs::Config; use crate::databases::Database; use crate::storages::StorageDescription; use crate::storages::Table; diff --git a/query/src/catalogs/impls/immutable_catalog.rs b/query/src/catalogs/impls/immutable_catalog.rs index 3a18837bc5d7..91be509b432b 100644 --- a/query/src/catalogs/impls/immutable_catalog.rs +++ b/query/src/catalogs/impls/immutable_catalog.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::CreateDatabaseReply; @@ -35,7 +36,6 @@ use crate::catalogs::catalog::Catalog; use crate::catalogs::InMemoryMetas; use crate::catalogs::SYS_DB_ID_BEGIN; use crate::catalogs::SYS_TBL_ID_BEGIN; -use crate::configs::Config; use crate::databases::Database; use crate::databases::InformationSchemaDatabase; use crate::databases::SystemDatabase; diff --git a/query/src/catalogs/impls/mutable_catalog.rs b/query/src/catalogs/impls/mutable_catalog.rs index 39bcbe4a120d..46bd65488dd7 100644 --- a/query/src/catalogs/impls/mutable_catalog.rs +++ b/query/src/catalogs/impls/mutable_catalog.rs @@ -15,9 +15,11 @@ use std::sync::Arc; +use common_configs::Config; use common_exception::Result; use common_meta_api::SchemaApi; use common_meta_embedded::MetaEmbedded; +use common_meta_grpc::MetaGrpcClientConf; use common_meta_types::CreateDatabaseReply; use common_meta_types::CreateDatabaseReq; use common_meta_types::CreateTableReq; @@ -46,7 +48,6 @@ use crate::catalogs::backends::MetaBackend; use crate::catalogs::catalog::Catalog; use crate::catalogs::CatalogContext; use crate::common::MetaClientProvider; -use crate::configs::Config; use crate::databases::Database; use crate::databases::DatabaseContext; use crate::databases::DatabaseFactory; @@ -91,8 +92,9 @@ impl MutableCatalog { } else { tracing::info!("use remote meta"); - let meta_client_provider = - Arc::new(MetaClientProvider::new(conf.meta.to_grpc_client_config())); + let meta_client_provider = Arc::new(MetaClientProvider::new(MetaGrpcClientConf::from( + &conf.meta, + ))); let meta_backend = MetaBackend::create(meta_client_provider); Arc::new(meta_backend) }; diff --git a/query/src/clusters/cluster.rs b/query/src/clusters/cluster.rs index 106f7d0219ad..9efe73ed0d17 100644 --- a/query/src/clusters/cluster.rs +++ b/query/src/clusters/cluster.rs @@ -28,12 +28,15 @@ use common_base::base::DummySignalStream; use common_base::base::GlobalUniqName; use common_base::base::SignalStream; use common_base::base::SignalType; +use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_grpc::ConnectionFactory; +use common_grpc::RpcClientTlsConfig; use common_management::ClusterApi; use common_management::ClusterMgr; use common_meta_api::KVApi; +use common_meta_grpc::MetaGrpcClientConf; use common_meta_types::NodeInfo; use common_tracing::tracing; use futures::future::select; @@ -45,7 +48,6 @@ use rand::Rng; use crate::api::FlightClient; use crate::common::MetaClientProvider; -use crate::configs::Config; pub struct ClusterDiscovery { local_id: String, @@ -55,7 +57,7 @@ pub struct ClusterDiscovery { impl ClusterDiscovery { async fn create_meta_client(cfg: &Config) -> Result> { - let meta_api_provider = MetaClientProvider::new(cfg.meta.to_grpc_client_config()); + let meta_api_provider = MetaClientProvider::new(MetaGrpcClientConf::from(&cfg.meta)); match meta_api_provider.try_get_kv_client().await { Ok(client) => Ok(client), Err(cause) => Err(cause.add_message_back("(while create cluster api).")), @@ -212,7 +214,7 @@ impl Cluster { ConnectionFactory::create_rpc_channel( node.flight_address.clone(), None, - Some(config.tls_query_client_conf()), + Some(RpcClientTlsConfig::from(&config.query)), )?, ))), false => Ok(FlightClient::new(FlightServiceClient::new( diff --git a/query/src/databases/database_factory.rs b/query/src/databases/database_factory.rs index efc974d3a419..eaeec723197b 100644 --- a/query/src/databases/database_factory.rs +++ b/query/src/databases/database_factory.rs @@ -17,11 +17,11 @@ use std::collections::HashMap; use std::sync::Arc; use common_base::infallible::RwLock; +use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::DatabaseInfo; -use crate::configs::Config; use crate::databases::default::DefaultDatabase; use crate::databases::github::GithubDatabase; use crate::databases::Database; diff --git a/query/src/databases/system/system_database.rs b/query/src/databases/system/system_database.rs index 47d4d922f10e..2f4e5d00298b 100644 --- a/query/src/databases/system/system_database.rs +++ b/query/src/databases/system/system_database.rs @@ -15,13 +15,13 @@ use std::sync::Arc; +use common_configs::Config; use common_meta_types::DatabaseIdent; use common_meta_types::DatabaseInfo; use common_meta_types::DatabaseMeta; use common_meta_types::DatabaseNameIdent; use crate::catalogs::InMemoryMetas; -use crate::configs::Config; use crate::databases::Database; use crate::storages::system; use crate::storages::Table; diff --git a/query/src/lib.rs b/query/src/lib.rs index d76db2afdd84..12ebf4b8ca88 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -23,7 +23,6 @@ pub mod api; pub mod catalogs; pub mod clusters; pub mod common; -pub mod configs; pub mod databases; pub mod formats; pub mod interpreters; diff --git a/query/src/servers/http/http_services.rs b/query/src/servers/http/http_services.rs index e4b909e71937..b59a89a02f51 100644 --- a/query/src/servers/http/http_services.rs +++ b/query/src/servers/http/http_services.rs @@ -16,6 +16,7 @@ use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; +use common_configs::Config; use common_exception::Result; use common_tracing::tracing; use poem::get; @@ -27,7 +28,6 @@ use poem::Route; use super::v1::upload_to_stage; use crate::common::service::HttpShutdownHandler; -use crate::configs::Config; use crate::servers::http::middleware::HTTPSessionMiddleware; use crate::servers::http::v1::clickhouse_router; use crate::servers::http::v1::query_route; diff --git a/query/src/servers/http/v1/query/http_query_manager.rs b/query/src/servers/http/v1/query/http_query_manager.rs index f4694cc573ff..6b7fe2318f96 100644 --- a/query/src/servers/http/v1/query/http_query_manager.rs +++ b/query/src/servers/http/v1/query/http_query_manager.rs @@ -20,12 +20,12 @@ use common_base::base::tokio; use common_base::base::tokio::sync::RwLock; use common_base::base::tokio::time::sleep; use common_base::infallible::Mutex; +use common_configs::Config; use common_exception::Result; use common_tracing::tracing; use super::expiring_map::ExpiringMap; use super::HttpQueryContext; -use crate::configs::Config; use crate::servers::http::v1::query::http_query::HttpQuery; use crate::servers::http::v1::query::HttpQueryRequest; use crate::sessions::SessionRef; diff --git a/query/src/servers/mysql/mysql_federated.rs b/query/src/servers/mysql/mysql_federated.rs index 4da53c307fb2..f80d8281afad 100644 --- a/query/src/servers/mysql/mysql_federated.rs +++ b/query/src/servers/mysql/mysql_federated.rs @@ -31,7 +31,7 @@ impl MySQLFederated { pub fn create() -> Self { MySQLFederated { mysql_version: MYSQL_VERSION.to_string(), - databend_version: crate::configs::DATABEND_COMMIT_VERSION.to_string(), + databend_version: common_configs::DATABEND_COMMIT_VERSION.to_string(), } } diff --git a/query/src/servers/mysql/mysql_interactive_worker.rs b/query/src/servers/mysql/mysql_interactive_worker.rs index b94c1885fc16..61b00e3ca783 100644 --- a/query/src/servers/mysql/mysql_interactive_worker.rs +++ b/query/src/servers/mysql/mysql_interactive_worker.rs @@ -442,7 +442,7 @@ impl InteractiveWorker { version: format!( "{}-{}", MYSQL_VERSION, - *crate::configs::DATABEND_COMMIT_VERSION + *common_configs::DATABEND_COMMIT_VERSION ), client_addr, } diff --git a/query/src/sessions/query_ctx.rs b/query/src/sessions/query_ctx.rs index 4e483d7d0e24..863de4d828f2 100644 --- a/query/src/sessions/query_ctx.rs +++ b/query/src/sessions/query_ctx.rs @@ -27,6 +27,7 @@ use common_base::base::Runtime; use common_base::base::TrySpawn; use common_base::infallible::Mutex; use common_base::infallible::RwLock; +use common_configs::Config; use common_contexts::DalContext; use common_contexts::DalMetrics; use common_datablocks::DataBlock; @@ -52,7 +53,6 @@ use opendal::Operator; use crate::catalogs::Catalog; use crate::catalogs::DatabaseCatalog; use crate::clusters::Cluster; -use crate::configs::Config; use crate::servers::http::v1::HttpQueryHandle; use crate::sessions::ProcessInfo; use crate::sessions::QueryContextShared; @@ -88,7 +88,7 @@ impl QueryContext { Arc::new(QueryContext { statistics: Arc::new(RwLock::new(Statistics::default())), partition_queue: Arc::new(RwLock::new(VecDeque::new())), - version: format!("DatabendQuery {}", *crate::configs::DATABEND_COMMIT_VERSION), + version: format!("DatabendQuery {}", *common_configs::DATABEND_COMMIT_VERSION), shared, precommit_blocks: Arc::new(RwLock::new(Vec::new())), }) diff --git a/query/src/sessions/query_ctx_shared.rs b/query/src/sessions/query_ctx_shared.rs index bfd1c3fd4d35..807925b31e63 100644 --- a/query/src/sessions/query_ctx_shared.rs +++ b/query/src/sessions/query_ctx_shared.rs @@ -22,6 +22,7 @@ use common_base::base::Progress; use common_base::base::Runtime; use common_base::infallible::Mutex; use common_base::infallible::RwLock; +use common_configs::Config; use common_contexts::DalContext; use common_exception::ErrorCode; use common_exception::Result; @@ -34,7 +35,6 @@ use uuid::Uuid; use crate::catalogs::Catalog; use crate::catalogs::DatabaseCatalog; use crate::clusters::Cluster; -use crate::configs::Config; use crate::servers::http::v1::HttpQueryHandle; use crate::sessions::Session; use crate::sessions::Settings; diff --git a/query/src/sessions/session.rs b/query/src/sessions/session.rs index 84d90f81d29b..b2f428131610 100644 --- a/query/src/sessions/session.rs +++ b/query/src/sessions/session.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use common_base::infallible::RwLock; use common_base::mem_allocator::malloc_size; +use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_macros::MallocSizeOf; @@ -28,7 +29,6 @@ use futures::channel::*; use opendal::Operator; use crate::catalogs::DatabaseCatalog; -use crate::configs::Config; use crate::sessions::QueryContext; use crate::sessions::QueryContextShared; use crate::sessions::SessionContext; diff --git a/query/src/sessions/session_ctx.rs b/query/src/sessions/session_ctx.rs index 22e4f879931f..622dc971670d 100644 --- a/query/src/sessions/session_ctx.rs +++ b/query/src/sessions/session_ctx.rs @@ -18,12 +18,12 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use common_base::infallible::RwLock; +use common_configs::Config; use common_exception::Result; use common_macros::MallocSizeOf; use common_meta_types::UserInfo; use futures::channel::oneshot::Sender; -use crate::configs::Config; use crate::sessions::QueryContextShared; #[derive(MallocSizeOf)] diff --git a/query/src/sessions/session_mgr.rs b/query/src/sessions/session_mgr.rs index 708c11054594..94f86d63c448 100644 --- a/query/src/sessions/session_mgr.rs +++ b/query/src/sessions/session_mgr.rs @@ -23,6 +23,7 @@ use common_base::base::tokio; use common_base::base::Runtime; use common_base::base::SignalStream; use common_base::infallible::RwLock; +use common_configs::Config; use common_contexts::DalRuntime; use common_exception::ErrorCode; use common_exception::Result; @@ -44,7 +45,6 @@ use opendal::Scheme as DalSchema; use crate::catalogs::DatabaseCatalog; use crate::clusters::ClusterDiscovery; -use crate::configs::Config; use crate::servers::http::v1::HttpQueryManager; use crate::sessions::session::Session; use crate::sessions::session_ref::SessionRef; diff --git a/query/src/sessions/session_settings.rs b/query/src/sessions/session_settings.rs index 099d85e9f0b7..5fc8fb5562c9 100644 --- a/query/src/sessions/session_settings.rs +++ b/query/src/sessions/session_settings.rs @@ -19,13 +19,13 @@ use std::fmt::Formatter; use std::sync::Arc; use common_base::infallible::RwLock; +use common_configs::Config; use common_datavalues::prelude::*; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::UserSetting; use itertools::Itertools; -use crate::configs::Config; use crate::sessions::SessionContext; use crate::users::UserApiProvider; diff --git a/query/src/storages/cache.rs b/query/src/storages/cache.rs index 0d69e861a875..ec53f5d4963b 100644 --- a/query/src/storages/cache.rs +++ b/query/src/storages/cache.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::configs::QueryConfig; +use common_configs::QueryConfig; + use crate::storages::fuse::cache; use crate::storages::fuse::cache::MemoryCache; use crate::storages::fuse::cache::SegmentInfoCache; diff --git a/query/src/storages/storage_factory.rs b/query/src/storages/storage_factory.rs index 4b27475ab6be..8e4e191878ce 100644 --- a/query/src/storages/storage_factory.rs +++ b/query/src/storages/storage_factory.rs @@ -16,11 +16,11 @@ use std::collections::HashMap; use std::sync::Arc; use common_base::infallible::RwLock; +use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::TableInfo; -use crate::configs::Config; use crate::storages::fuse::FuseTable; use crate::storages::github::GithubTable; use crate::storages::memory::MemoryTable; diff --git a/query/src/users/auth/auth_mgr.rs b/query/src/users/auth/auth_mgr.rs index 7887c59d8948..6bc64b8fa401 100644 --- a/query/src/users/auth/auth_mgr.rs +++ b/query/src/users/auth/auth_mgr.rs @@ -14,13 +14,13 @@ use std::sync::Arc; +pub use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::AuthInfo; use common_meta_types::UserIdentity; use common_meta_types::UserInfo; -pub use crate::configs::Config; use crate::users::auth::jwt::JwtAuthenticator; use crate::users::UserApiProvider; diff --git a/query/src/users/auth/jwt/authenticator.rs b/query/src/users/auth/jwt/authenticator.rs index 5e1d3725e447..b256e38185c3 100644 --- a/query/src/users/auth/jwt/authenticator.rs +++ b/query/src/users/auth/jwt/authenticator.rs @@ -14,6 +14,7 @@ use std::time::Duration; +use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use jwtk::jwk::RemoteJwksVerifier; @@ -21,8 +22,6 @@ use jwtk::HeaderAndClaims; use serde::Deserialize; use serde::Serialize; -use crate::configs::Config; - pub struct JwtAuthenticator { //Todo(youngsofun): verify settings, like issuer verifier: RemoteJwksVerifier, diff --git a/query/src/users/user_api.rs b/query/src/users/user_api.rs index 2d84763257b7..cbc2ca5b3cd2 100644 --- a/query/src/users/user_api.rs +++ b/query/src/users/user_api.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use common_configs::Config; use common_exception::Result; use common_management::RoleApi; use common_management::RoleMgr; @@ -26,9 +27,9 @@ use common_management::UdfMgr; use common_management::UserApi; use common_management::UserMgr; use common_meta_api::KVApi; +use common_meta_grpc::MetaGrpcClientConf; use crate::common::MetaClientProvider; -use crate::configs::Config; pub struct UserApiProvider { client: Arc, @@ -36,7 +37,7 @@ pub struct UserApiProvider { impl UserApiProvider { pub async fn create_global(conf: Config) -> Result> { - let client = MetaClientProvider::new(conf.meta.to_grpc_client_config()) + let client = MetaClientProvider::new(MetaGrpcClientConf::from(&conf.meta)) .try_get_kv_client() .await?; diff --git a/query/tests/it/main.rs b/query/tests/it/main.rs index 41670b9fa564..1520d0789e19 100644 --- a/query/tests/it/main.rs +++ b/query/tests/it/main.rs @@ -15,7 +15,6 @@ mod api; mod catalogs; mod clusters; mod common; -mod configs; mod formats; mod functions; mod interpreters; diff --git a/query/tests/it/sessions/query_ctx.rs b/query/tests/it/sessions/query_ctx.rs index 4d53032c3676..217b8449f209 100644 --- a/query/tests/it/sessions/query_ctx.rs +++ b/query/tests/it/sessions/query_ctx.rs @@ -13,9 +13,9 @@ // limitations under the License. use common_base::base::tokio; +use common_configs::FsStorageConfig; +use common_configs::S3StorageConfig; use common_exception::Result; -use databend_query::configs::FsStorageConfig; -use databend_query::configs::S3StorageConfig; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] // This test need network diff --git a/query/tests/it/sessions/session_context.rs b/query/tests/it/sessions/session_context.rs index 53d6c832a735..0003553291eb 100644 --- a/query/tests/it/sessions/session_context.rs +++ b/query/tests/it/sessions/session_context.rs @@ -17,10 +17,10 @@ use std::net::SocketAddr; use std::sync::Arc; use common_base::base::tokio; +use common_configs::Config; use common_exception::Result; use common_meta_types::UserInfo; use databend_query::clusters::Cluster; -use databend_query::configs::Config; use databend_query::sessions::QueryContextShared; use databend_query::sessions::SessionContext; use databend_query::sessions::SessionType; diff --git a/query/tests/it/tests/config.rs b/query/tests/it/tests/config.rs index e092720dd9dc..992b4beba2f6 100644 --- a/query/tests/it/tests/config.rs +++ b/query/tests/it/tests/config.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_query::configs::Config; +use common_configs::Config; pub struct ConfigBuilder { conf: Config, diff --git a/query/tests/it/tests/context.rs b/query/tests/it/tests/context.rs index 25327f0b9819..423fc3718018 100644 --- a/query/tests/it/tests/context.rs +++ b/query/tests/it/tests/context.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use common_configs::Config; use common_exception::Result; use common_meta_embedded::MetaEmbedded; use common_meta_types::AuthInfo; @@ -23,7 +24,6 @@ use common_meta_types::PasswordHashMethod; use common_meta_types::UserInfo; use common_meta_types::UserPrivilegeSet; use databend_query::clusters::Cluster; -use databend_query::configs::Config; use databend_query::sessions::QueryContext; use databend_query::sessions::QueryContextShared; use databend_query::sessions::SessionType; diff --git a/query/tests/it/tests/sessions.rs b/query/tests/it/tests/sessions.rs index 4421593e0d48..e58d0294dcdf 100644 --- a/query/tests/it/tests/sessions.rs +++ b/query/tests/it/tests/sessions.rs @@ -17,8 +17,8 @@ use std::sync::Arc; use common_base::base::tokio::runtime::Runtime; use common_base::base::Thread; +use common_configs::Config; use common_exception::Result; -use databend_query::configs::Config; use databend_query::sessions::SessionManager; async fn async_create_sessions(config: Config) -> Result> { From ed7688ed003c1ae41390cf7d6da21df6e17f2cb7 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Tue, 10 May 2022 11:40:51 +0800 Subject: [PATCH 11/14] delete dbg --- query/src/sql/exec/data_schema_builder.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/query/src/sql/exec/data_schema_builder.rs b/query/src/sql/exec/data_schema_builder.rs index 721bcde688f4..fae8e215616d 100644 --- a/query/src/sql/exec/data_schema_builder.rs +++ b/query/src/sql/exec/data_schema_builder.rs @@ -74,7 +74,6 @@ impl<'a> DataSchemaBuilder<'a> { let column_entry = self.metadata.column(*index); let field_name = format_field_name(column_entry.name.as_str(), *index); let field = DataField::new(field_name.as_str(), column_entry.data_type.clone()); - dbg!(field.clone()); fields.push(field); } @@ -86,9 +85,7 @@ impl<'a> DataSchemaBuilder<'a> { for index in columns { let column_entry = self.metadata.column(*index); let field_name = column_entry.name.clone(); - let field = - DataField::new_nullable(field_name.as_str(), column_entry.data_type.clone()); - + let field = DataField::new(field_name.as_str(), column_entry.data_type.clone()); fields.push(field); } From 6ea91e651bf3e5a4b9cef6454de1552bc55c4075 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 10 May 2022 12:01:25 +0800 Subject: [PATCH 12/14] Format toml Signed-off-by: Xuanwo --- common/configs/Cargo.toml | 4 ++-- common/grpc/Cargo.toml | 2 +- common/meta/grpc/Cargo.toml | 2 +- query/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/configs/Cargo.toml b/common/configs/Cargo.toml index 39d397c3c7d8..b21b45c39df0 100644 --- a/common/configs/Cargo.toml +++ b/common/configs/Cargo.toml @@ -11,17 +11,17 @@ test = false [dependencies] # Workspace dependencies common-base = { path = "../../common/base" } -common-exception = {path = "../../common/exception"} +common-exception = { path = "../../common/exception" } # Github dependencies # Crates.io dependencies clap = { version = "3.1.8", features = ["derive", "env"] } +once_cell = "1.10.0" serde = { version = "1.0.136", features = ["derive"] } serde-bridge = "0.0.3" serde_json = "1.0.79" serfig = "0.0.1" -once_cell = "1.10.0" [dev-dependencies] pretty_assertions = "1.2.1" diff --git a/common/grpc/Cargo.toml b/common/grpc/Cargo.toml index bbb2e19ef602..7a975dd0bd36 100644 --- a/common/grpc/Cargo.toml +++ b/common/grpc/Cargo.toml @@ -13,9 +13,9 @@ test = false [dependencies] # In alphabetical order # Workspace dependencies common-base = { path = "../base" } +common-configs = { path = "../configs" } common-exception = { path = "../exception" } common-tracing = { path = "../tracing" } -common-configs = {path = "../configs"} # Github dependencies diff --git a/common/meta/grpc/Cargo.toml b/common/meta/grpc/Cargo.toml index 00a3ae05d096..fb05c8745584 100644 --- a/common/meta/grpc/Cargo.toml +++ b/common/meta/grpc/Cargo.toml @@ -16,6 +16,7 @@ test = false [dependencies] common-arrow = { path = "../../arrow" } common-base = { path = "../../base" } +common-configs = { path = "../../configs" } common-exception = { path = "../../exception" } common-grpc = { path = "../../grpc" } common-meta-api = { path = "../api" } @@ -23,7 +24,6 @@ common-meta-types = { path = "../types" } common-proto-conv = { path = "../../proto-conv" } common-protos = { path = "../../protos" } common-tracing = { path = "../../tracing" } -common-configs = {path = "../../configs"} derive_more = "0.99.17" futures = "0.3.21" diff --git a/query/Cargo.toml b/query/Cargo.toml index a5ad42f516c6..fabb4a5dc64e 100644 --- a/query/Cargo.toml +++ b/query/Cargo.toml @@ -30,6 +30,7 @@ common-arrow = { path = "../common/arrow" } common-ast = { path = "../common/ast" } common-base = { path = "../common/base" } common-cache = { path = "../common/cache" } +common-configs = { path = "../common/configs" } common-contexts = { path = "../common/contexts" } common-datablocks = { path = "../common/datablocks" } common-datavalues = { path = "../common/datavalues" } @@ -49,7 +50,6 @@ common-metrics = { path = "../common/metrics" } common-planners = { path = "../common/planners" } common-streams = { path = "../common/streams" } common-tracing = { path = "../common/tracing" } -common-configs = {path ="../common/configs"} # Github dependencies bincode = { git = "https://github.com/datafuse-extras/bincode", rev = "fd3f9ff" } From fb81d8277563e29919d3a6fd2e62300ed6ead90c Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 10 May 2022 12:45:27 +0800 Subject: [PATCH 13/14] Fix databend version Signed-off-by: Xuanwo --- Cargo.lock | 1 - common/configs/Cargo.toml | 1 - common/configs/src/config.rs | 19 ----------- common/configs/src/lib.rs | 9 +++--- common/configs/tests/configs.rs | 7 ---- query/src/bin/databend-query.rs | 2 +- query/src/lib.rs | 3 ++ query/src/servers/mysql/mysql_federated.rs | 2 +- .../servers/mysql/mysql_interactive_worker.rs | 2 +- query/src/sessions/query_ctx.rs | 2 +- query/src/version.rs | 32 +++++++++++++++++++ query/tests/it/main.rs | 1 + query/tests/it/version.rs | 21 ++++++++++++ 13 files changed, 65 insertions(+), 37 deletions(-) create mode 100644 query/src/version.rs create mode 100644 query/tests/it/version.rs diff --git a/Cargo.lock b/Cargo.lock index 4be37f2f57f8..29954862ae1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -910,7 +910,6 @@ dependencies = [ "clap 3.1.8", "common-base", "common-exception", - "once_cell", "pretty_assertions", "serde", "serde-bridge", diff --git a/common/configs/Cargo.toml b/common/configs/Cargo.toml index b21b45c39df0..44fdbca0517d 100644 --- a/common/configs/Cargo.toml +++ b/common/configs/Cargo.toml @@ -17,7 +17,6 @@ common-exception = { path = "../../common/exception" } # Crates.io dependencies clap = { version = "3.1.8", features = ["derive", "env"] } -once_cell = "1.10.0" serde = { version = "1.0.136", features = ["derive"] } serde-bridge = "0.0.3" serde_json = "1.0.79" diff --git a/common/configs/src/config.rs b/common/configs/src/config.rs index 52b4c8251189..dd4f1c3122fc 100644 --- a/common/configs/src/config.rs +++ b/common/configs/src/config.rs @@ -16,7 +16,6 @@ use std::env; use clap::Parser; use common_exception::Result; -use once_cell::sync::Lazy; use serde::Deserialize; use serde::Serialize; use serfig::collectors::from_env; @@ -29,24 +28,6 @@ use crate::MetaConfig; use crate::QueryConfig; use crate::StorageConfig; -pub static DATABEND_COMMIT_VERSION: Lazy = Lazy::new(|| { - let git_tag = option_env!("VERGEN_GIT_SEMVER"); - let git_sha = option_env!("VERGEN_GIT_SHA_SHORT"); - let rustc_semver = option_env!("VERGEN_RUSTC_SEMVER"); - let timestamp = option_env!("VERGEN_BUILD_TIMESTAMP"); - - let ver = match (git_tag, git_sha, rustc_semver, timestamp) { - #[cfg(not(feature = "simd"))] - (Some(v1), Some(v2), Some(v3), Some(v4)) => format!("{}-{}(rust-{}-{})", v1, v2, v3, v4), - #[cfg(feature = "simd")] - (Some(v1), Some(v2), Some(v3), Some(v4)) => { - format!("{}-{}-simd(rust-{}-{})", v1, v2, v3, v4) - } - _ => String::new(), - }; - ver -}); - #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize, Parser)] #[clap(about, version, author)] #[serde(default)] diff --git a/common/configs/src/lib.rs b/common/configs/src/lib.rs index 3529bd155a8d..ed8f93697b7b 100644 --- a/common/configs/src/lib.rs +++ b/common/configs/src/lib.rs @@ -13,13 +13,12 @@ // limitations under the License. mod config; -pub mod log; -pub mod meta; -pub mod query; -pub mod storage; +mod log; +mod meta; +mod query; +mod storage; pub use config::Config; -pub use config::DATABEND_COMMIT_VERSION; pub use log::LogConfig; pub use meta::MetaConfig; pub use query::QueryConfig; diff --git a/common/configs/tests/configs.rs b/common/configs/tests/configs.rs index 69d53494bd6f..6e98f131ca42 100644 --- a/common/configs/tests/configs.rs +++ b/common/configs/tests/configs.rs @@ -300,10 +300,3 @@ root = "" Ok(()) } - -#[test] -fn test_fuse_commit_version() -> Result<()> { - let v = &common_configs::DATABEND_COMMIT_VERSION; - assert!(v.len() > 0); - Ok(()) -} diff --git a/query/src/bin/databend-query.rs b/query/src/bin/databend-query.rs index bca1a3c74846..05792ee8f169 100644 --- a/query/src/bin/databend-query.rs +++ b/query/src/bin/databend-query.rs @@ -55,7 +55,7 @@ async fn main(_global_tracker: Arc) -> common_exception::Result< set_panic_hook(); tracing::info!("{:?}", conf); - tracing::info!("DatabendQuery {}", *common_configs::DATABEND_COMMIT_VERSION); + tracing::info!("DatabendQuery {}", *databend_query::DATABEND_COMMIT_VERSION); let session_manager = SessionManager::from_conf(conf.clone()).await?; let mut shutdown_handle = ShutdownHandle::create(session_manager.clone()); diff --git a/query/src/lib.rs b/query/src/lib.rs index 12ebf4b8ca88..722e1796872e 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -36,3 +36,6 @@ pub mod sql; pub mod storages; pub mod table_functions; pub mod users; + +mod version; +pub use version::DATABEND_COMMIT_VERSION; diff --git a/query/src/servers/mysql/mysql_federated.rs b/query/src/servers/mysql/mysql_federated.rs index f80d8281afad..921e50f4085c 100644 --- a/query/src/servers/mysql/mysql_federated.rs +++ b/query/src/servers/mysql/mysql_federated.rs @@ -31,7 +31,7 @@ impl MySQLFederated { pub fn create() -> Self { MySQLFederated { mysql_version: MYSQL_VERSION.to_string(), - databend_version: common_configs::DATABEND_COMMIT_VERSION.to_string(), + databend_version: crate::version::DATABEND_COMMIT_VERSION.to_string(), } } diff --git a/query/src/servers/mysql/mysql_interactive_worker.rs b/query/src/servers/mysql/mysql_interactive_worker.rs index 61b00e3ca783..370964f7cfe2 100644 --- a/query/src/servers/mysql/mysql_interactive_worker.rs +++ b/query/src/servers/mysql/mysql_interactive_worker.rs @@ -442,7 +442,7 @@ impl InteractiveWorker { version: format!( "{}-{}", MYSQL_VERSION, - *common_configs::DATABEND_COMMIT_VERSION + *crate::version::DATABEND_COMMIT_VERSION ), client_addr, } diff --git a/query/src/sessions/query_ctx.rs b/query/src/sessions/query_ctx.rs index 863de4d828f2..11e77dd5a078 100644 --- a/query/src/sessions/query_ctx.rs +++ b/query/src/sessions/query_ctx.rs @@ -88,7 +88,7 @@ impl QueryContext { Arc::new(QueryContext { statistics: Arc::new(RwLock::new(Statistics::default())), partition_queue: Arc::new(RwLock::new(VecDeque::new())), - version: format!("DatabendQuery {}", *common_configs::DATABEND_COMMIT_VERSION), + version: format!("DatabendQuery {}", *crate::version::DATABEND_COMMIT_VERSION), shared, precommit_blocks: Arc::new(RwLock::new(Vec::new())), }) diff --git a/query/src/version.rs b/query/src/version.rs new file mode 100644 index 000000000000..7f00590fd63a --- /dev/null +++ b/query/src/version.rs @@ -0,0 +1,32 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use once_cell::sync::Lazy; + +pub static DATABEND_COMMIT_VERSION: Lazy = Lazy::new(|| { + let git_tag = option_env!("VERGEN_GIT_SEMVER"); + let git_sha = option_env!("VERGEN_GIT_SHA_SHORT"); + let rustc_semver = option_env!("VERGEN_RUSTC_SEMVER"); + let timestamp = option_env!("VERGEN_BUILD_TIMESTAMP"); + + let ver = match (git_tag, git_sha, rustc_semver, timestamp) { + #[cfg(not(feature = "simd"))] + (Some(v1), Some(v2), Some(v3), Some(v4)) => format!("{}-{}(rust-{}-{})", v1, v2, v3, v4), + #[cfg(feature = "simd")] + (Some(v1), Some(v2), Some(v3), Some(v4)) => { + format!("{}-{}-simd(rust-{}-{})", v1, v2, v3, v4) + } + _ => String::new(), + }; + ver +}); diff --git a/query/tests/it/main.rs b/query/tests/it/main.rs index 1520d0789e19..23890043e420 100644 --- a/query/tests/it/main.rs +++ b/query/tests/it/main.rs @@ -28,3 +28,4 @@ mod storages; mod table_functions; mod tests; mod users; +mod version; diff --git a/query/tests/it/version.rs b/query/tests/it/version.rs new file mode 100644 index 000000000000..098e4176fa98 --- /dev/null +++ b/query/tests/it/version.rs @@ -0,0 +1,21 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use common_exception::Result; + +#[test] +fn test_databend_commit_version() -> Result<()> { + let v = &databend_query::DATABEND_COMMIT_VERSION; + assert!(v.len() > 0); + Ok(()) +} From 9c530c2d97283d7f6e8123e1f365c1e51abc322b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 10 May 2022 13:03:40 +0800 Subject: [PATCH 14/14] Keep config under query Signed-off-by: Xuanwo --- Cargo.lock | 7 +------ common/configs/Cargo.toml | 8 -------- common/configs/src/lib.rs | 2 -- common/configs/tests/main.rs | 14 -------------- query/Cargo.toml | 1 + query/benches/suites/mod.rs | 2 +- query/src/api/http_service.rs | 2 +- query/src/api/rpc_service.rs | 2 +- query/src/bin/databend-query.rs | 2 +- query/src/catalogs/impls/database_catalog.rs | 2 +- query/src/catalogs/impls/immutable_catalog.rs | 2 +- query/src/catalogs/impls/mutable_catalog.rs | 2 +- query/src/clusters/cluster.rs | 2 +- {common/configs => query}/src/config.rs | 9 ++++----- query/src/databases/database_factory.rs | 2 +- query/src/databases/system/system_database.rs | 2 +- query/src/lib.rs | 3 +++ query/src/servers/http/http_services.rs | 2 +- .../servers/http/v1/query/http_query_manager.rs | 2 +- query/src/sessions/query_ctx.rs | 2 +- query/src/sessions/query_ctx_shared.rs | 2 +- query/src/sessions/session.rs | 2 +- query/src/sessions/session_ctx.rs | 2 +- query/src/sessions/session_mgr.rs | 2 +- query/src/sessions/session_settings.rs | 2 +- query/src/storages/storage_factory.rs | 2 +- query/src/users/auth/auth_mgr.rs | 2 +- query/src/users/auth/jwt/authenticator.rs | 3 ++- query/src/users/user_api.rs | 2 +- .../configs/tests => query/tests/it}/configs.rs | 2 +- query/tests/it/main.rs | 1 + query/tests/it/sessions/session_context.rs | 2 +- query/tests/it/tests/config.rs | 2 +- query/tests/it/tests/context.rs | 2 +- query/tests/it/tests/sessions.rs | 2 +- 35 files changed, 38 insertions(+), 62 deletions(-) delete mode 100644 common/configs/tests/main.rs rename {common/configs => query}/src/config.rs (95%) rename {common/configs/tests => query/tests/it}/configs.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 29954862ae1f..9958837acd69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -910,13 +910,7 @@ dependencies = [ "clap 3.1.8", "common-base", "common-exception", - "pretty_assertions", "serde", - "serde-bridge", - "serde_json", - "serfig", - "temp-env", - "toml", ] [[package]] @@ -1840,6 +1834,7 @@ dependencies = [ "sha1", "sha2 0.10.2", "sqlparser", + "temp-env", "tempfile", "thiserror", "threadpool", diff --git a/common/configs/Cargo.toml b/common/configs/Cargo.toml index 44fdbca0517d..ca697326e38d 100644 --- a/common/configs/Cargo.toml +++ b/common/configs/Cargo.toml @@ -18,11 +18,3 @@ common-exception = { path = "../../common/exception" } # Crates.io dependencies clap = { version = "3.1.8", features = ["derive", "env"] } serde = { version = "1.0.136", features = ["derive"] } -serde-bridge = "0.0.3" -serde_json = "1.0.79" -serfig = "0.0.1" - -[dev-dependencies] -pretty_assertions = "1.2.1" -toml = { version = "0.5.8", default-features = false } -temp-env = "0.2.0" diff --git a/common/configs/src/lib.rs b/common/configs/src/lib.rs index ed8f93697b7b..4a2a9682ff92 100644 --- a/common/configs/src/lib.rs +++ b/common/configs/src/lib.rs @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod config; mod log; mod meta; mod query; mod storage; -pub use config::Config; pub use log::LogConfig; pub use meta::MetaConfig; pub use query::QueryConfig; diff --git a/common/configs/tests/main.rs b/common/configs/tests/main.rs deleted file mode 100644 index de888d3a63e7..000000000000 --- a/common/configs/tests/main.rs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2022 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -mod configs; diff --git a/query/Cargo.toml b/query/Cargo.toml index fabb4a5dc64e..9be4f8dba374 100644 --- a/query/Cargo.toml +++ b/query/Cargo.toml @@ -124,6 +124,7 @@ maplit = "1.0.2" mysql_async = "0.29.0" pretty_assertions = "1.2.1" reqwest = { version = "0.11.10", features = ["json", "native-tls"] } +temp-env = "0.2.0" tempfile = "3.3.0" toml = { version = "0.5.8", default-features = false } url = "2.2.2" diff --git a/query/benches/suites/mod.rs b/query/benches/suites/mod.rs index 32f07e786a84..66761a957260 100644 --- a/query/benches/suites/mod.rs +++ b/query/benches/suites/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. use common_base::base::tokio; -use common_configs::Config; use common_exception::Result; use common_planners::PlanNode; use criterion::Criterion; @@ -21,6 +20,7 @@ use databend_query::interpreters::SelectInterpreter; use databend_query::sessions::SessionManager; use databend_query::sessions::SessionType; use databend_query::sql::PlanParser; +use databend_query::Config; use futures::StreamExt; pub mod bench_aggregate_query_sql; diff --git a/query/src/api/http_service.rs b/query/src/api/http_service.rs index cb6aa09ffc7d..cfbb5fac096b 100644 --- a/query/src/api/http_service.rs +++ b/query/src/api/http_service.rs @@ -16,7 +16,6 @@ use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; -use common_configs::Config; use common_exception::Result; use common_tracing::tracing; use poem::get; @@ -28,6 +27,7 @@ use poem::Route; use crate::common::service::HttpShutdownHandler; use crate::servers::Server; use crate::sessions::SessionManager; +use crate::Config; pub struct HttpService { sessions: Arc, diff --git a/query/src/api/rpc_service.rs b/query/src/api/rpc_service.rs index 1778c6b6b53a..1b95ce4bab84 100644 --- a/query/src/api/rpc_service.rs +++ b/query/src/api/rpc_service.rs @@ -20,7 +20,6 @@ use common_arrow::arrow_format::flight::service::flight_service_server::FlightSe use common_base::base::tokio; use common_base::base::tokio::net::TcpListener; use common_base::base::tokio::sync::Notify; -use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_tracing::tracing; @@ -33,6 +32,7 @@ use crate::api::rpc::DatabendQueryFlightDispatcher; use crate::api::rpc::DatabendQueryFlightService; use crate::servers::Server as DatabendQueryServer; use crate::sessions::SessionManager; +use crate::Config; pub struct RpcService { pub sessions: Arc, diff --git a/query/src/bin/databend-query.rs b/query/src/bin/databend-query.rs index 05792ee8f169..c7bcecbc1b56 100644 --- a/query/src/bin/databend-query.rs +++ b/query/src/bin/databend-query.rs @@ -15,7 +15,6 @@ use std::sync::Arc; use common_base::base::RuntimeTracker; -use common_configs::Config; use common_macros::databend_main; use common_meta_embedded::MetaEmbedded; use common_metrics::init_default_metrics_recorder; @@ -31,6 +30,7 @@ use databend_query::servers::MySQLHandler; use databend_query::servers::Server; use databend_query::servers::ShutdownHandle; use databend_query::sessions::SessionManager; +use databend_query::Config; #[databend_main] async fn main(_global_tracker: Arc) -> common_exception::Result<()> { diff --git a/query/src/catalogs/impls/database_catalog.rs b/query/src/catalogs/impls/database_catalog.rs index f4f61f90e1b1..4f61fb817827 100644 --- a/query/src/catalogs/impls/database_catalog.rs +++ b/query/src/catalogs/impls/database_catalog.rs @@ -15,7 +15,6 @@ use std::sync::Arc; -use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::CreateDatabaseReply; @@ -43,6 +42,7 @@ use crate::storages::Table; use crate::table_functions::TableArgs; use crate::table_functions::TableFunction; use crate::table_functions::TableFunctionFactory; +use crate::Config; /// Combine two catalogs together /// - read/search like operations are always performed at diff --git a/query/src/catalogs/impls/immutable_catalog.rs b/query/src/catalogs/impls/immutable_catalog.rs index 91be509b432b..8b7bd8143d9e 100644 --- a/query/src/catalogs/impls/immutable_catalog.rs +++ b/query/src/catalogs/impls/immutable_catalog.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::CreateDatabaseReply; @@ -40,6 +39,7 @@ use crate::databases::Database; use crate::databases::InformationSchemaDatabase; use crate::databases::SystemDatabase; use crate::storages::Table; +use crate::Config; /// System Catalog contains ... all the system databases (no surprise :) /// Currently, this is only one database here, the "system" db. diff --git a/query/src/catalogs/impls/mutable_catalog.rs b/query/src/catalogs/impls/mutable_catalog.rs index 46bd65488dd7..5fe9f8cfeba2 100644 --- a/query/src/catalogs/impls/mutable_catalog.rs +++ b/query/src/catalogs/impls/mutable_catalog.rs @@ -15,7 +15,6 @@ use std::sync::Arc; -use common_configs::Config; use common_exception::Result; use common_meta_api::SchemaApi; use common_meta_embedded::MetaEmbedded; @@ -55,6 +54,7 @@ use crate::storages::StorageContext; use crate::storages::StorageDescription; use crate::storages::StorageFactory; use crate::storages::Table; +use crate::Config; /// Catalog based on MetaStore /// - System Database NOT included diff --git a/query/src/clusters/cluster.rs b/query/src/clusters/cluster.rs index 9efe73ed0d17..59da3fd53641 100644 --- a/query/src/clusters/cluster.rs +++ b/query/src/clusters/cluster.rs @@ -28,7 +28,6 @@ use common_base::base::DummySignalStream; use common_base::base::GlobalUniqName; use common_base::base::SignalStream; use common_base::base::SignalType; -use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_grpc::ConnectionFactory; @@ -48,6 +47,7 @@ use rand::Rng; use crate::api::FlightClient; use crate::common::MetaClientProvider; +use crate::Config; pub struct ClusterDiscovery { local_id: String, diff --git a/common/configs/src/config.rs b/query/src/config.rs similarity index 95% rename from common/configs/src/config.rs rename to query/src/config.rs index dd4f1c3122fc..45593991bca6 100644 --- a/common/configs/src/config.rs +++ b/query/src/config.rs @@ -15,6 +15,10 @@ use std::env; use clap::Parser; +use common_configs::LogConfig; +use common_configs::MetaConfig; +use common_configs::QueryConfig; +use common_configs::StorageConfig; use common_exception::Result; use serde::Deserialize; use serde::Serialize; @@ -23,11 +27,6 @@ use serfig::collectors::from_file; use serfig::collectors::from_self; use serfig::parsers::Toml; -use crate::LogConfig; -use crate::MetaConfig; -use crate::QueryConfig; -use crate::StorageConfig; - #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize, Parser)] #[clap(about, version, author)] #[serde(default)] diff --git a/query/src/databases/database_factory.rs b/query/src/databases/database_factory.rs index eaeec723197b..e577571ba518 100644 --- a/query/src/databases/database_factory.rs +++ b/query/src/databases/database_factory.rs @@ -17,7 +17,6 @@ use std::collections::HashMap; use std::sync::Arc; use common_base::infallible::RwLock; -use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::DatabaseInfo; @@ -26,6 +25,7 @@ use crate::databases::default::DefaultDatabase; use crate::databases::github::GithubDatabase; use crate::databases::Database; use crate::databases::DatabaseContext; +use crate::Config; pub trait DatabaseCreator: Send + Sync { fn try_create(&self, ctx: DatabaseContext, db_info: DatabaseInfo) -> Result>; diff --git a/query/src/databases/system/system_database.rs b/query/src/databases/system/system_database.rs index 2f4e5d00298b..55a582e123ec 100644 --- a/query/src/databases/system/system_database.rs +++ b/query/src/databases/system/system_database.rs @@ -15,7 +15,6 @@ use std::sync::Arc; -use common_configs::Config; use common_meta_types::DatabaseIdent; use common_meta_types::DatabaseInfo; use common_meta_types::DatabaseMeta; @@ -25,6 +24,7 @@ use crate::catalogs::InMemoryMetas; use crate::databases::Database; use crate::storages::system; use crate::storages::Table; +use crate::Config; #[derive(Clone)] pub struct SystemDatabase { diff --git a/query/src/lib.rs b/query/src/lib.rs index 722e1796872e..1fad921e94e5 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -37,5 +37,8 @@ pub mod storages; pub mod table_functions; pub mod users; +mod config; mod version; + +pub use config::Config; pub use version::DATABEND_COMMIT_VERSION; diff --git a/query/src/servers/http/http_services.rs b/query/src/servers/http/http_services.rs index b59a89a02f51..c77f7a01ab60 100644 --- a/query/src/servers/http/http_services.rs +++ b/query/src/servers/http/http_services.rs @@ -16,7 +16,6 @@ use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; -use common_configs::Config; use common_exception::Result; use common_tracing::tracing; use poem::get; @@ -34,6 +33,7 @@ use crate::servers::http::v1::query_route; use crate::servers::http::v1::streaming_load; use crate::servers::Server; use crate::sessions::SessionManager; +use crate::Config; pub struct HttpHandler { session_manager: Arc, diff --git a/query/src/servers/http/v1/query/http_query_manager.rs b/query/src/servers/http/v1/query/http_query_manager.rs index 6b7fe2318f96..8fef1dbbf4e2 100644 --- a/query/src/servers/http/v1/query/http_query_manager.rs +++ b/query/src/servers/http/v1/query/http_query_manager.rs @@ -20,7 +20,6 @@ use common_base::base::tokio; use common_base::base::tokio::sync::RwLock; use common_base::base::tokio::time::sleep; use common_base::infallible::Mutex; -use common_configs::Config; use common_exception::Result; use common_tracing::tracing; @@ -29,6 +28,7 @@ use super::HttpQueryContext; use crate::servers::http::v1::query::http_query::HttpQuery; use crate::servers::http::v1::query::HttpQueryRequest; use crate::sessions::SessionRef; +use crate::Config; // TODO(youngsofun): may need refactor later for 2 reasons: // 1. some can be both configured and overwritten by http query request diff --git a/query/src/sessions/query_ctx.rs b/query/src/sessions/query_ctx.rs index 11e77dd5a078..86f4889b1a66 100644 --- a/query/src/sessions/query_ctx.rs +++ b/query/src/sessions/query_ctx.rs @@ -27,7 +27,6 @@ use common_base::base::Runtime; use common_base::base::TrySpawn; use common_base::infallible::Mutex; use common_base::infallible::RwLock; -use common_configs::Config; use common_contexts::DalContext; use common_contexts::DalMetrics; use common_datablocks::DataBlock; @@ -65,6 +64,7 @@ use crate::storages::Table; use crate::users::auth::auth_mgr::AuthMgr; use crate::users::RoleCacheMgr; use crate::users::UserApiProvider; +use crate::Config; #[derive(Clone)] pub struct QueryContext { diff --git a/query/src/sessions/query_ctx_shared.rs b/query/src/sessions/query_ctx_shared.rs index 807925b31e63..51c63bea0efe 100644 --- a/query/src/sessions/query_ctx_shared.rs +++ b/query/src/sessions/query_ctx_shared.rs @@ -22,7 +22,6 @@ use common_base::base::Progress; use common_base::base::Runtime; use common_base::infallible::Mutex; use common_base::infallible::RwLock; -use common_configs::Config; use common_contexts::DalContext; use common_exception::ErrorCode; use common_exception::Result; @@ -43,6 +42,7 @@ use crate::storages::Table; use crate::users::auth::auth_mgr::AuthMgr; use crate::users::RoleCacheMgr; use crate::users::UserApiProvider; +use crate::Config; type DatabaseAndTable = (String, String); diff --git a/query/src/sessions/session.rs b/query/src/sessions/session.rs index b2f428131610..9fc6ace7572e 100644 --- a/query/src/sessions/session.rs +++ b/query/src/sessions/session.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use common_base::infallible::RwLock; use common_base::mem_allocator::malloc_size; -use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_macros::MallocSizeOf; @@ -36,6 +35,7 @@ use crate::sessions::SessionManager; use crate::sessions::SessionStatus; use crate::sessions::SessionType; use crate::sessions::Settings; +use crate::Config; #[derive(Clone, MallocSizeOf)] pub struct Session { diff --git a/query/src/sessions/session_ctx.rs b/query/src/sessions/session_ctx.rs index 622dc971670d..8f43500921c3 100644 --- a/query/src/sessions/session_ctx.rs +++ b/query/src/sessions/session_ctx.rs @@ -18,13 +18,13 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use common_base::infallible::RwLock; -use common_configs::Config; use common_exception::Result; use common_macros::MallocSizeOf; use common_meta_types::UserInfo; use futures::channel::oneshot::Sender; use crate::sessions::QueryContextShared; +use crate::Config; #[derive(MallocSizeOf)] pub struct SessionContext { diff --git a/query/src/sessions/session_mgr.rs b/query/src/sessions/session_mgr.rs index 94f86d63c448..be46c37043b2 100644 --- a/query/src/sessions/session_mgr.rs +++ b/query/src/sessions/session_mgr.rs @@ -23,7 +23,6 @@ use common_base::base::tokio; use common_base::base::Runtime; use common_base::base::SignalStream; use common_base::infallible::RwLock; -use common_configs::Config; use common_contexts::DalRuntime; use common_exception::ErrorCode; use common_exception::Result; @@ -54,6 +53,7 @@ use crate::sessions::SessionType; use crate::storages::cache::CacheManager; use crate::users::auth::auth_mgr::AuthMgr; use crate::users::UserApiProvider; +use crate::Config; pub struct SessionManager { pub(in crate::sessions) conf: RwLock, diff --git a/query/src/sessions/session_settings.rs b/query/src/sessions/session_settings.rs index 5fc8fb5562c9..56be358d736e 100644 --- a/query/src/sessions/session_settings.rs +++ b/query/src/sessions/session_settings.rs @@ -19,7 +19,6 @@ use std::fmt::Formatter; use std::sync::Arc; use common_base::infallible::RwLock; -use common_configs::Config; use common_datavalues::prelude::*; use common_exception::ErrorCode; use common_exception::Result; @@ -28,6 +27,7 @@ use itertools::Itertools; use crate::sessions::SessionContext; use crate::users::UserApiProvider; +use crate::Config; #[derive(Clone)] enum ScopeLevel { diff --git a/query/src/storages/storage_factory.rs b/query/src/storages/storage_factory.rs index 8e4e191878ce..4829a902a6f0 100644 --- a/query/src/storages/storage_factory.rs +++ b/query/src/storages/storage_factory.rs @@ -16,7 +16,6 @@ use std::collections::HashMap; use std::sync::Arc; use common_base::infallible::RwLock; -use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::TableInfo; @@ -28,6 +27,7 @@ use crate::storages::null::NullTable; use crate::storages::view::ViewTable; use crate::storages::StorageContext; use crate::storages::Table; +use crate::Config; pub trait StorageCreator: Send + Sync { fn try_create(&self, ctx: StorageContext, table_info: TableInfo) -> Result>; diff --git a/query/src/users/auth/auth_mgr.rs b/query/src/users/auth/auth_mgr.rs index 6bc64b8fa401..eea1a5e5ffc5 100644 --- a/query/src/users/auth/auth_mgr.rs +++ b/query/src/users/auth/auth_mgr.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -pub use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use common_meta_types::AuthInfo; @@ -23,6 +22,7 @@ use common_meta_types::UserInfo; use crate::users::auth::jwt::JwtAuthenticator; use crate::users::UserApiProvider; +pub use crate::Config; pub struct AuthMgr { tenant: String, diff --git a/query/src/users/auth/jwt/authenticator.rs b/query/src/users/auth/jwt/authenticator.rs index b256e38185c3..985c6562af8e 100644 --- a/query/src/users/auth/jwt/authenticator.rs +++ b/query/src/users/auth/jwt/authenticator.rs @@ -14,7 +14,6 @@ use std::time::Duration; -use common_configs::Config; use common_exception::ErrorCode; use common_exception::Result; use jwtk::jwk::RemoteJwksVerifier; @@ -22,6 +21,8 @@ use jwtk::HeaderAndClaims; use serde::Deserialize; use serde::Serialize; +use crate::Config; + pub struct JwtAuthenticator { //Todo(youngsofun): verify settings, like issuer verifier: RemoteJwksVerifier, diff --git a/query/src/users/user_api.rs b/query/src/users/user_api.rs index cbc2ca5b3cd2..53008aeaf3b9 100644 --- a/query/src/users/user_api.rs +++ b/query/src/users/user_api.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use common_configs::Config; use common_exception::Result; use common_management::RoleApi; use common_management::RoleMgr; @@ -30,6 +29,7 @@ use common_meta_api::KVApi; use common_meta_grpc::MetaGrpcClientConf; use crate::common::MetaClientProvider; +use crate::Config; pub struct UserApiProvider { client: Arc, diff --git a/common/configs/tests/configs.rs b/query/tests/it/configs.rs similarity index 99% rename from common/configs/tests/configs.rs rename to query/tests/it/configs.rs index 6e98f131ca42..ca11eccd3460 100644 --- a/common/configs/tests/configs.rs +++ b/query/tests/it/configs.rs @@ -16,8 +16,8 @@ use std::env::temp_dir; use std::fs; use std::io::Write; -use common_configs::Config; use common_exception::Result; +use databend_query::Config; use pretty_assertions::assert_eq; // Default. diff --git a/query/tests/it/main.rs b/query/tests/it/main.rs index 23890043e420..3506e234fb88 100644 --- a/query/tests/it/main.rs +++ b/query/tests/it/main.rs @@ -15,6 +15,7 @@ mod api; mod catalogs; mod clusters; mod common; +mod configs; mod formats; mod functions; mod interpreters; diff --git a/query/tests/it/sessions/session_context.rs b/query/tests/it/sessions/session_context.rs index 0003553291eb..3d11560bf43d 100644 --- a/query/tests/it/sessions/session_context.rs +++ b/query/tests/it/sessions/session_context.rs @@ -17,13 +17,13 @@ use std::net::SocketAddr; use std::sync::Arc; use common_base::base::tokio; -use common_configs::Config; use common_exception::Result; use common_meta_types::UserInfo; use databend_query::clusters::Cluster; use databend_query::sessions::QueryContextShared; use databend_query::sessions::SessionContext; use databend_query::sessions::SessionType; +use databend_query::Config; use crate::tests::SessionManagerBuilder; diff --git a/query/tests/it/tests/config.rs b/query/tests/it/tests/config.rs index 992b4beba2f6..d02090512cce 100644 --- a/query/tests/it/tests/config.rs +++ b/query/tests/it/tests/config.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use common_configs::Config; +use databend_query::Config; pub struct ConfigBuilder { conf: Config, diff --git a/query/tests/it/tests/context.rs b/query/tests/it/tests/context.rs index 423fc3718018..119e1454ff52 100644 --- a/query/tests/it/tests/context.rs +++ b/query/tests/it/tests/context.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use common_configs::Config; use common_exception::Result; use common_meta_embedded::MetaEmbedded; use common_meta_types::AuthInfo; @@ -28,6 +27,7 @@ use databend_query::sessions::QueryContext; use databend_query::sessions::QueryContextShared; use databend_query::sessions::SessionType; use databend_query::storages::StorageContext; +use databend_query::Config; use crate::tests::SessionManagerBuilder; diff --git a/query/tests/it/tests/sessions.rs b/query/tests/it/tests/sessions.rs index e58d0294dcdf..0634d99662d7 100644 --- a/query/tests/it/tests/sessions.rs +++ b/query/tests/it/tests/sessions.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use common_base::base::tokio::runtime::Runtime; use common_base::base::Thread; -use common_configs::Config; use common_exception::Result; use databend_query::sessions::SessionManager; +use databend_query::Config; async fn async_create_sessions(config: Config) -> Result> { let sessions = SessionManager::from_conf(config.clone()).await?;