Merge remote-tracking branch 'origin/main' into feat-multi-catalog

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "common/base",
     "common/building",
     "common/cache",
+    "common/configs",
     "common/contexts",
     "common/datablocks",
     "common/datavalues",

common/configs/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "common-configs"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+doctest = false
+test = false
+
+[dependencies]
+# Workspace dependencies
+common-base = { path = "../../common/base" }
+common-exception = { path = "../../common/exception" }
+
+# Github dependencies
+
+# Crates.io dependencies
+clap = { version = "3.1.8", features = ["derive", "env"] }
+serde = { version = "1.0.136", features = ["derive"] }

query/src/configs/mod.rs → common/configs/src/lib.rs

@@ -12,20 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod config;
-pub mod config_catalog;
-pub mod config_log;
-pub mod config_meta;
-pub mod config_query;
-pub mod config_storage;
+mod catalog;
+mod log;
+mod meta;
+mod query;
+mod storage;
 
-pub use config::Config;
-pub use config::DATABEND_COMMIT_VERSION;
-pub use config_catalog::HiveCatalogConfig;
-pub use config_log::LogConfig;
-pub use config_meta::MetaConfig;
-pub use config_query::QueryConfig;
-pub use config_storage::AzblobStorageConfig;
-pub use config_storage::FsStorageConfig;
-pub use config_storage::S3StorageConfig;
-pub use config_storage::StorageConfig;
+pub use catalog::HiveCatalogConfig;
+pub use catalog::ThriftProtocol;
+pub use log::LogConfig;
+pub use meta::MetaConfig;
+pub use query::QueryConfig;
+pub use storage::AzblobStorageConfig;
+pub use storage::FsStorageConfig;
+pub use storage::S3StorageConfig;
+pub use storage::StorageConfig;

query/src/configs/config_meta.rs → common/configs/src/meta.rs

@@ -16,9 +16,6 @@ use std::fmt;
 
 use clap::Args;
 use common_base::base::mask_string;
-use common_grpc::RpcClientConf;
-use common_grpc::RpcClientTlsConfig;
-use common_meta_grpc::MetaGrpcClientConf;
 use serde::Deserialize;
 use serde::Serialize;
 
@@ -79,32 +76,6 @@ impl MetaConfig {
         !self.rpc_tls_meta_server_root_ca_cert.is_empty()
             && !self.rpc_tls_meta_service_domain_name.is_empty()
     }
-
-    pub fn to_grpc_tls_config(&self) -> Option<RpcClientTlsConfig> {
-        if !self.is_tls_enabled() {
-            return None;
-        }
-
-        Some(RpcClientTlsConfig {
-            rpc_tls_server_root_ca_cert: self.rpc_tls_meta_server_root_ca_cert.clone(),
-            domain_name: self.rpc_tls_meta_service_domain_name.clone(),
-        })
-    }
-
-    pub fn to_grpc_client_config(&self) -> MetaGrpcClientConf {
-        let meta_config = RpcClientConf {
-            address: self.address.clone(),
-            username: self.username.clone(),
-            password: self.password.clone(),
-            tls_conf: self.to_grpc_tls_config(),
-        };
-
-        MetaGrpcClientConf {
-            meta_service_config: meta_config.clone(),
-            kv_service_config: meta_config,
-            client_timeout_in_second: self.client_timeout_in_second,
-        }
-    }
 }
 
 impl fmt::Debug for MetaConfig {

common/grpc/Cargo.toml

@@ -13,6 +13,7 @@ test = false
 [dependencies] # In alphabetical order
 # Workspace dependencies
 common-base = { path = "../base" }
+common-configs = { path = "../configs" }
 common-exception = { path = "../exception" }
 common-tracing = { path = "../tracing" }
 

common/grpc/src/client_conf.rs

@@ -13,6 +13,9 @@
 //  limitations under the License.
 //
 
+use common_configs::MetaConfig;
+use common_configs::QueryConfig;
+
 #[derive(Clone, Debug, Default)]
 pub struct RpcClientTlsConfig {
     pub rpc_tls_server_root_ca_cert: String,
@@ -25,6 +28,24 @@ impl RpcClientTlsConfig {
     }
 }
 
+impl From<&QueryConfig> for RpcClientTlsConfig {
+    fn from(qc: &QueryConfig) -> Self {
+        RpcClientTlsConfig {
+            rpc_tls_server_root_ca_cert: qc.rpc_tls_query_server_root_ca_cert.to_string(),
+            domain_name: qc.rpc_tls_query_service_domain_name.to_string(),
+        }
+    }
+}
+
+impl From<&MetaConfig> for RpcClientTlsConfig {
+    fn from(mc: &MetaConfig) -> Self {
+        RpcClientTlsConfig {
+            rpc_tls_server_root_ca_cert: mc.rpc_tls_meta_server_root_ca_cert.to_string(),
+            domain_name: mc.rpc_tls_meta_service_domain_name.to_string(),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Default)]
 pub struct RpcClientConf {
     pub address: String,

common/meta/grpc/Cargo.toml

@@ -16,6 +16,7 @@ test = false
 [dependencies]
 common-arrow = { path = "../../arrow" }
 common-base = { path = "../../base" }
+common-configs = { path = "../../configs" }
 common-exception = { path = "../../exception" }
 common-grpc = { path = "../../grpc" }
 common-meta-api = { path = "../api" }

common/meta/grpc/src/grpc_client_conf.rs

@@ -12,11 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_configs::MetaConfig;
 use common_grpc::RpcClientConf;
+use common_grpc::RpcClientTlsConfig;
 
 #[derive(Clone, Debug, Default)]
 pub struct MetaGrpcClientConf {
     pub meta_service_config: RpcClientConf,
     pub kv_service_config: RpcClientConf,
     pub client_timeout_in_second: u64,
 }
+
+impl From<&MetaConfig> for MetaGrpcClientConf {
+    fn from(mc: &MetaConfig) -> Self {
+        let meta_config = RpcClientConf {
+            address: mc.address.clone(),
+            username: mc.username.clone(),
+            password: mc.password.clone(),
+            tls_conf: if mc.is_tls_enabled() {
+                Some(RpcClientTlsConfig::from(mc))
+            } else {
+                None
+            },
+        };
+
+        MetaGrpcClientConf {
+            meta_service_config: meta_config.clone(),
+            kv_service_config: meta_config,
+            client_timeout_in_second: mc.client_timeout_in_second,
+        }
+    }
+}

query/Cargo.toml

@@ -31,6 +31,7 @@ common-arrow = { path = "../common/arrow" }
 common-ast = { path = "../common/ast" }
 common-base = { path = "../common/base" }
 common-cache = { path = "../common/cache" }
+common-configs = { path = "../common/configs" }
 common-contexts = { path = "../common/contexts" }
 common-datablocks = { path = "../common/datablocks" }
 common-datavalues = { path = "../common/datavalues" }

query/benches/suites/mod.rs

@@ -16,11 +16,11 @@ use common_base::base::tokio;
 use common_exception::Result;
 use common_planners::PlanNode;
 use criterion::Criterion;
-use databend_query::configs::Config;
 use databend_query::interpreters::SelectInterpreter;
 use databend_query::sessions::SessionManager;
 use databend_query::sessions::SessionType;
 use databend_query::sql::PlanParser;
+use databend_query::Config;
 use futures::StreamExt;
 
 pub mod bench_aggregate_query_sql;

query/src/api/http_service.rs

@@ -25,9 +25,9 @@ use poem::EndpointExt;
 use poem::Route;
 
 use crate::common::service::HttpShutdownHandler;
-use crate::configs::Config;
 use crate::servers::Server;
 use crate::sessions::SessionManager;
+use crate::Config;
 
 pub struct HttpService {
     sessions: Arc<SessionManager>,

query/src/api/rpc_service.rs

@@ -30,9 +30,9 @@ use tonic::transport::ServerTlsConfig;
 
 use crate::api::rpc::DatabendQueryFlightDispatcher;
 use crate::api::rpc::DatabendQueryFlightService;
-use crate::configs::Config;
 use crate::servers::Server as DatabendQueryServer;
 use crate::sessions::SessionManager;
+use crate::Config;
 
 pub struct RpcService {
     pub sessions: Arc<SessionManager>,

query/src/bin/databend-query.rs

@@ -23,14 +23,14 @@ use common_tracing::set_panic_hook;
 use common_tracing::tracing;
 use databend_query::api::HttpService;
 use databend_query::api::RpcService;
-use databend_query::configs::Config;
 use databend_query::metrics::MetricService;
 use databend_query::servers::ClickHouseHandler;
 use databend_query::servers::HttpHandler;
 use databend_query::servers::MySQLHandler;
 use databend_query::servers::Server;
 use databend_query::servers::ShutdownHandle;
 use databend_query::sessions::SessionManager;
+use databend_query::Config;
 
 #[databend_main]
 async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<()> {
@@ -55,10 +55,7 @@ async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<
 
     set_panic_hook();
     tracing::info!("{:?}", conf);
-    tracing::info!(
-        "DatabendQuery {}",
-        *databend_query::configs::DATABEND_COMMIT_VERSION,
-    );
+    tracing::info!("DatabendQuery {}", *databend_query::DATABEND_COMMIT_VERSION);
 
     let session_manager = SessionManager::from_conf(conf.clone()).await?;
     let mut shutdown_handle = ShutdownHandle::create(session_manager.clone());

query/src/catalogs/catalog_manager.rs

@@ -22,7 +22,7 @@ use crate::catalogs::default::DatabaseCatalog;
 #[cfg(feature = "hive")]
 use crate::catalogs::hive::HiveCatalog;
 use crate::catalogs::Catalog;
-use crate::configs::Config;
+use crate::config::Config;
 
 // TODO catalogs are hard coded
 

query/src/catalogs/default/database_catalog.rs

@@ -33,16 +33,16 @@ use common_meta_types::UpsertTableOptionReply;
 use common_meta_types::UpsertTableOptionReq;
 use common_tracing::tracing;
 
-use super::immutable_catalog::ImmutableCatalog;
-use super::mutable_catalog::MutableCatalog;
 use crate::catalogs::catalog::Catalog;
-use crate::configs::Config;
+use crate::catalogs::default::ImmutableCatalog;
+use crate::catalogs::default::MutableCatalog;
 use crate::databases::Database;
 use crate::storages::StorageDescription;
 use crate::storages::Table;
 use crate::table_functions::TableArgs;
 use crate::table_functions::TableFunction;
 use crate::table_functions::TableFunctionFactory;
+use crate::Config;
 
 /// Combine two catalogs together
 /// - read/search like operations are always performed at
@@ -59,7 +59,7 @@ pub struct DatabaseCatalog {
 }
 
 impl DatabaseCatalog {
-    fn create(
+    pub fn create(
         immutable_catalog: Arc<dyn Catalog>,
         mutable_catalog: Arc<dyn Catalog>,
         table_function_factory: Arc<TableFunctionFactory>,

query/src/catalogs/default/immutable_catalog.rs

@@ -31,15 +31,15 @@ use common_meta_types::TableMeta;
 use common_meta_types::UpsertTableOptionReply;
 use common_meta_types::UpsertTableOptionReq;
 
-use super::table_id_ranges::SYS_DB_ID_BEGIN;
-use super::table_id_ranges::SYS_TBL_ID_BEGIN;
-use super::table_memory_meta::InMemoryMetas;
 use crate::catalogs::catalog::Catalog;
-use crate::configs::Config;
+use crate::catalogs::InMemoryMetas;
+use crate::catalogs::SYS_DB_ID_BEGIN;
+use crate::catalogs::SYS_TBL_ID_BEGIN;
 use crate::databases::Database;
 use crate::databases::InformationSchemaDatabase;
 use crate::databases::SystemDatabase;
 use crate::storages::Table;
+use crate::Config;
 
 /// System Catalog contains ... all the system databases (no surprise :)
 /// Currently, this is only one database here, the "system" db.

query/src/catalogs/default/mutable_catalog.rs

@@ -18,6 +18,7 @@ use std::sync::Arc;
 use common_exception::Result;
 use common_meta_api::SchemaApi;
 use common_meta_embedded::MetaEmbedded;
+use common_meta_grpc::MetaGrpcClientConf;
 use common_meta_types::CreateDatabaseReply;
 use common_meta_types::CreateDatabaseReq;
 use common_meta_types::CreateTableReq;
@@ -46,7 +47,6 @@ use super::backends::MetaBackend;
 use super::catalog_context::CatalogContext;
 use crate::catalogs::catalog::Catalog;
 use crate::common::MetaClientProvider;
-use crate::configs::Config;
 use crate::databases::Database;
 use crate::databases::DatabaseContext;
 use crate::databases::DatabaseFactory;
@@ -55,6 +55,7 @@ use crate::storages::StorageContext;
 use crate::storages::StorageDescription;
 use crate::storages::StorageFactory;
 use crate::storages::Table;
+use crate::Config;
 
 /// Catalog based on MetaStore
 /// - System Database NOT included
@@ -94,8 +95,9 @@ impl MutableCatalog {
         } else {
             tracing::info!("use remote meta");
 
-            let meta_client_provider =
-                Arc::new(MetaClientProvider::new(conf.meta.to_grpc_client_config()));
+            let meta_client_provider = Arc::new(MetaClientProvider::new(MetaGrpcClientConf::from(
+                &conf.meta,
+            )));
             let meta_backend = MetaBackend::create(meta_client_provider);
             Arc::new(meta_backend)
         };