From a44da11832d3e0d24a14096b34232a85f2ab89ce Mon Sep 17 00:00:00 2001 From: psvri Date: Fri, 26 Aug 2022 19:32:07 +0000 Subject: [PATCH 1/2] Add show external tables --- benchmarks/src/bin/h2o.rs | 2 +- benchmarks/src/bin/tpch.rs | 2 +- .../examples/parquet_sql_multiple_files.rs | 1 + .../core/benches/sort_limit_query_sql.rs | 2 +- datafusion/core/src/catalog/schema.rs | 2 +- .../core/src/datasource/listing/table.rs | 21 +++++++---- datafusion/core/src/execution/context.rs | 36 +++++++++++++------ datafusion/core/tests/path_partition.rs | 4 +-- .../core/tests/sql/information_schema.rs | 23 ++++++++++++ datafusion/expr/src/logical_plan/plan.rs | 17 +++++++++ datafusion/proto/proto/datafusion.proto | 1 + datafusion/proto/src/logical_plan.rs | 12 ++++++- datafusion/sql/src/parser.rs | 14 +++++++- datafusion/sql/src/planner.rs | 2 ++ 14 files changed, 115 insertions(+), 24 deletions(-) diff --git a/benchmarks/src/bin/h2o.rs b/benchmarks/src/bin/h2o.rs index 88f4084e87a5..b0bff885baaa 100644 --- a/benchmarks/src/bin/h2o.rs +++ b/benchmarks/src/bin/h2o.rs @@ -83,7 +83,7 @@ async fn group_by(opt: &GroupBy) -> Result<()> { let listing_config = ListingTableConfig::new(ListingTableUrl::parse(path)?) .with_listing_options(ListingOptions::new(Arc::new(CsvFormat::default()))) .with_schema(Arc::new(schema)); - let csv = ListingTable::try_new(listing_config)?; + let csv = ListingTable::try_new(listing_config, None)?; let partition_size = num_cpus::get(); let memtable = MemTable::load(Arc::new(csv), Some(partition_size), &ctx.state()).await?; diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs index 08374bfcc5b2..a6ae9c61d1e1 100644 --- a/benchmarks/src/bin/tpch.rs +++ b/benchmarks/src/bin/tpch.rs @@ -429,7 +429,7 @@ fn get_table( .with_listing_options(options) .with_schema(schema); - Ok(Arc::new(ListingTable::try_new(config)?)) + Ok(Arc::new(ListingTable::try_new(config, None)?)) } fn get_schema(table: &str) -> Schema { diff --git a/datafusion-examples/examples/parquet_sql_multiple_files.rs b/datafusion-examples/examples/parquet_sql_multiple_files.rs index b948f16835ac..2b7a5f6fe268 100644 --- a/datafusion-examples/examples/parquet_sql_multiple_files.rs +++ b/datafusion-examples/examples/parquet_sql_multiple_files.rs @@ -49,6 +49,7 @@ async fn main() -> Result<()> { &format!("file://{}", testdata), listing_options, None, + None, ) .await .unwrap(); diff --git a/datafusion/core/benches/sort_limit_query_sql.rs b/datafusion/core/benches/sort_limit_query_sql.rs index e7aa33bd70bd..95b4c6592669 100644 --- a/datafusion/core/benches/sort_limit_query_sql.rs +++ b/datafusion/core/benches/sort_limit_query_sql.rs @@ -74,7 +74,7 @@ fn create_context() -> Arc> { .with_listing_options(listing_options) .with_schema(schema); - let csv = async { ListingTable::try_new(config).unwrap() }; + let csv = async { ListingTable::try_new(config, None).unwrap() }; let rt = Runtime::new().unwrap(); diff --git a/datafusion/core/src/catalog/schema.rs b/datafusion/core/src/catalog/schema.rs index b886966bf6db..8ce064c74477 100644 --- a/datafusion/core/src/catalog/schema.rs +++ b/datafusion/core/src/catalog/schema.rs @@ -184,7 +184,7 @@ mod tests { .infer(&ctx.state()) .await .unwrap(); - let table = ListingTable::try_new(config).unwrap(); + let table = ListingTable::try_new(config, None).unwrap(); schema .register_table("alltypes_plain".to_string(), Arc::new(table)) diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index abf5394fb643..ee741a795515 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -246,6 +246,7 @@ pub struct ListingTable { /// File fields + partition columns table_schema: SchemaRef, options: ListingOptions, + definition: Option, } impl ListingTable { @@ -256,7 +257,10 @@ impl ListingTable { /// If the schema is provided then it must be resolved before creating the table /// and should contain the fields of the file without the table /// partitioning columns. - pub fn try_new(config: ListingTableConfig) -> Result { + pub fn try_new( + config: ListingTableConfig, + definition: Option, + ) -> Result { let file_schema = config .file_schema .ok_or_else(|| DataFusionError::Internal("No schema provided.".into()))?; @@ -280,6 +284,7 @@ impl ListingTable { file_schema, table_schema: Arc::new(Schema::new(table_fields)), options, + definition, }; Ok(table) @@ -358,6 +363,10 @@ impl TableProvider for ListingTable { Ok(TableProviderFilterPushDown::Inexact) } } + + fn get_table_definition(&self) -> Option<&str> { + self.definition.as_deref() + } } impl ListingTable { @@ -460,7 +469,7 @@ mod tests { let config = ListingTableConfig::new(table_path) .with_listing_options(opt) .with_schema(schema); - let table = ListingTable::try_new(config)?; + let table = ListingTable::try_new(config, None)?; let exec = table.scan(&state, &None, &[], None).await?; assert_eq!(exec.statistics().num_rows, Some(8)); @@ -489,7 +498,7 @@ mod tests { let config = ListingTableConfig::new(table_path) .with_listing_options(opt) .with_schema(file_schema); - let table = ListingTable::try_new(config)?; + let table = ListingTable::try_new(config, None)?; assert_eq!( columns(&table.schema()), @@ -660,7 +669,7 @@ mod tests { let config = ListingTableConfig::new(table_path) .infer(&ctx.state()) .await?; - let table = ListingTable::try_new(config)?; + let table = ListingTable::try_new(config, None)?; Ok(Arc::new(table)) } @@ -692,7 +701,7 @@ mod tests { .with_listing_options(opt) .with_schema(Arc::new(schema)); - let table = ListingTable::try_new(config)?; + let table = ListingTable::try_new(config, None)?; let (file_list, _) = table.list_files_for_scan(&ctx.state(), &[], None).await?; @@ -732,7 +741,7 @@ mod tests { .with_listing_options(opt) .with_schema(Arc::new(schema)); - let table = ListingTable::try_new(config)?; + let table = ListingTable::try_new(config, None)?; let (file_list, _) = table.list_files_for_scan(&ctx.state(), &[], None).await?; diff --git a/datafusion/core/src/execution/context.rs b/datafusion/core/src/execution/context.rs index a0dee62de78f..7794a421b674 100644 --- a/datafusion/core/src/execution/context.rs +++ b/datafusion/core/src/execution/context.rs @@ -244,6 +244,7 @@ impl SessionContext { ref delimiter, ref table_partition_cols, ref if_not_exists, + definition, }) => { let (file_format, file_extension) = match file_type { FileType::CSV => ( @@ -292,6 +293,7 @@ impl SessionContext { location, options, provided_schema, + definition, ) .await?; let plan = LogicalPlanBuilder::empty(false).build()?; @@ -558,7 +560,7 @@ impl SessionContext { let config = ListingTableConfig::new(table_path) .with_listing_options(listing_options) .with_schema(resolved_schema); - let provider = ListingTable::try_new(config)?; + let provider = ListingTable::try_new(config, None)?; self.read_table(Arc::new(provider)) } @@ -584,7 +586,7 @@ impl SessionContext { let config = ListingTableConfig::new(table_path) .with_listing_options(listing_options) .with_schema(resolved_schema); - let provider = ListingTable::try_new(config)?; + let provider = ListingTable::try_new(config, None)?; self.read_table(Arc::new(provider)) } @@ -618,7 +620,7 @@ impl SessionContext { .with_listing_options(listing_options) .with_schema(resolved_schema); - let provider = ListingTable::try_new(config)?; + let provider = ListingTable::try_new(config, None)?; self.read_table(Arc::new(provider)) } @@ -642,7 +644,7 @@ impl SessionContext { .with_listing_options(listing_options) .with_schema(resolved_schema); - let provider = ListingTable::try_new(config)?; + let provider = ListingTable::try_new(config, None)?; self.read_table(Arc::new(provider)) } @@ -664,6 +666,7 @@ impl SessionContext { table_path: impl AsRef, options: ListingOptions, provided_schema: Option, + sql: Option, ) -> Result<()> { let table_path = ListingTableUrl::parse(table_path)?; let resolved_schema = match provided_schema { @@ -673,7 +676,7 @@ impl SessionContext { let config = ListingTableConfig::new(table_path) .with_listing_options(options) .with_schema(resolved_schema); - let table = ListingTable::try_new(config)?; + let table = ListingTable::try_new(config, sql)?; self.register_table(name, Arc::new(table))?; Ok(()) } @@ -694,6 +697,7 @@ impl SessionContext { table_path, listing_options, options.schema.map(|s| Arc::new(s.to_owned())), + None, ) .await?; @@ -711,8 +715,14 @@ impl SessionContext { let listing_options = options.to_listing_options(self.copied_config().target_partitions); - self.register_listing_table(name, table_path, listing_options, options.schema) - .await?; + self.register_listing_table( + name, + table_path, + listing_options, + options.schema, + None, + ) + .await?; Ok(()) } @@ -732,7 +742,7 @@ impl SessionContext { .parquet_pruning(parquet_pruning) .to_listing_options(target_partitions); - self.register_listing_table(name, table_path, listing_options, None) + self.register_listing_table(name, table_path, listing_options, None, None) .await?; Ok(()) } @@ -748,8 +758,14 @@ impl SessionContext { let listing_options = options.to_listing_options(self.copied_config().target_partitions); - self.register_listing_table(name, table_path, listing_options, options.schema) - .await?; + self.register_listing_table( + name, + table_path, + listing_options, + options.schema, + None, + ) + .await?; Ok(()) } diff --git a/datafusion/core/tests/path_partition.rs b/datafusion/core/tests/path_partition.rs index fca9b9a43b1c..4aee7b672410 100644 --- a/datafusion/core/tests/path_partition.rs +++ b/datafusion/core/tests/path_partition.rs @@ -441,7 +441,7 @@ fn register_partitioned_aggregate_csv( let config = ListingTableConfig::new(table_path) .with_listing_options(options) .with_schema(file_schema); - let table = ListingTable::try_new(config).unwrap(); + let table = ListingTable::try_new(config, None).unwrap(); ctx.register_table("t", Arc::new(table)) .expect("registering listing table failed"); @@ -479,7 +479,7 @@ async fn register_partitioned_alltypes_parquet( .with_listing_options(options) .with_schema(file_schema); - let table = ListingTable::try_new(config).unwrap(); + let table = ListingTable::try_new(config, None).unwrap(); ctx.register_table("t", Arc::new(table)) .expect("registering listing table failed"); diff --git a/datafusion/core/tests/sql/information_schema.rs b/datafusion/core/tests/sql/information_schema.rs index 5a4aa297f808..017330e44f33 100644 --- a/datafusion/core/tests/sql/information_schema.rs +++ b/datafusion/core/tests/sql/information_schema.rs @@ -605,6 +605,29 @@ async fn show_create_table() { assert_batches_eq!(expected, &results); } +#[tokio::test] +async fn show_external_create_table() { + let ctx = + SessionContext::with_config(SessionConfig::new().with_information_schema(true)); + + let table_sql = + "CREATE EXTERNAL TABLE abc STORED AS CSV WITH HEADER ROW LOCATION '../../testing/data/csv/aggregate_test_100.csv'"; + plan_and_collect(&ctx, table_sql).await.unwrap(); + + let result_sql = "SHOW CREATE TABLE abc"; + let results = plan_and_collect(&ctx, result_sql).await.unwrap(); + + let expected = vec![ + "+---------------+--------------+------------+-------------------------------------------------------------------------------------------------+", + "| table_catalog | table_schema | table_name | definition |", + "+---------------+--------------+------------+-------------------------------------------------------------------------------------------------+", + "| datafusion | public | abc | CREATE EXTERNAL TABLE abc STORED AS CSV LOCATION ../../testing/data/csv/aggregate_test_100.csv |", + "+---------------+--------------+------------+-------------------------------------------------------------------------------------------------+", + ]; + + assert_batches_eq!(expected, &results); +} + /// Execute SQL and return results async fn plan_and_collect(ctx: &SessionContext, sql: &str) -> Result> { ctx.sql(sql).await?.collect().await diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 2d5eb46804df..6d7b2d962f68 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -1227,6 +1227,21 @@ pub enum FileType { Avro, } +impl fmt::Display for FileType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + FileType::NdJson => "NDJSON", + FileType::Parquet => "PARQUET", + FileType::CSV => "CSV", + FileType::Avro => "AVRO", + } + ) + } +} + /// Creates an external table. #[derive(Clone)] pub struct CreateExternalTable { @@ -1246,6 +1261,8 @@ pub struct CreateExternalTable { pub table_partition_cols: Vec, /// Option to not error if table already exists pub if_not_exists: bool, + /// SQL used to create the view, if available + pub definition: Option, } /// Produces a relation with string representations of diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index 0b4a43e83e71..1ad8dec49f74 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -153,6 +153,7 @@ message CreateExternalTableNode { repeated string table_partition_cols = 6; bool if_not_exists = 7; string delimiter = 8; + string definition = 9; } message CreateCatalogSchemaNode { diff --git a/datafusion/proto/src/logical_plan.rs b/datafusion/proto/src/logical_plan.rs index bcdb9796f17e..ef2725639d86 100644 --- a/datafusion/proto/src/logical_plan.rs +++ b/datafusion/proto/src/logical_plan.rs @@ -430,7 +430,7 @@ impl AsLogicalPlan for LogicalPlanNode { .with_listing_options(options) .with_schema(Arc::new(schema)); - let provider = ListingTable::try_new(config)?; + let provider = ListingTable::try_new(config, None)?; LogicalPlanBuilder::scan_with_filters( &scan.table_name, @@ -493,6 +493,11 @@ impl AsLogicalPlan for LogicalPlanNode { let pb_file_type: protobuf::FileType = create_extern_table.file_type.try_into()?; + let definition = if !create_extern_table.definition.is_empty() { + Some(create_extern_table.definition.clone()) + } else { + None + }; Ok(LogicalPlan::CreateExternalTable(CreateExternalTable { schema: pb_schema.try_into()?, @@ -507,6 +512,7 @@ impl AsLogicalPlan for LogicalPlanNode { .table_partition_cols .clone(), if_not_exists: create_extern_table.if_not_exists, + definition, })) } LogicalPlanType::CreateView(create_view) => { @@ -1055,6 +1061,7 @@ impl AsLogicalPlan for LogicalPlanNode { schema: df_schema, table_partition_cols, if_not_exists, + definition, }) => { use datafusion::logical_plan::FileType; @@ -1076,6 +1083,9 @@ impl AsLogicalPlan for LogicalPlanNode { table_partition_cols: table_partition_cols.clone(), if_not_exists: *if_not_exists, delimiter: String::from(*delimiter), + definition: definition + .clone() + .unwrap_or_else(|| "".to_string()), }, )), }) diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index 28f27b390d7c..5b8639411a0e 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -26,7 +26,7 @@ use sqlparser::{ parser::{Parser, ParserError}, tokenizer::{Token, Tokenizer}, }; -use std::collections::VecDeque; +use std::{collections::VecDeque, fmt}; // Use `Parser::expected` instead, if possible macro_rules! parser_err { @@ -69,6 +69,18 @@ pub struct CreateExternalTable { pub if_not_exists: bool, } +impl fmt::Display for CreateExternalTable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "CREATE EXTERNAL TABLE ")?; + if self.if_not_exists { + write!(f, "IF NOT EXSISTS ")?; + } + write!(f, "{} ", self.name)?; + write!(f, "STORED AS {} ", self.file_type)?; + write!(f, "LOCATION {} ", self.location) + } +} + /// DataFusion extension DDL for `DESCRIBE TABLE` #[derive(Debug, Clone, PartialEq, Eq)] pub struct DescribeTable { diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index 094b980f1f85..7f1d3a9e9ffd 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -443,6 +443,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, statement: CreateExternalTable, ) -> Result { + let definition = Some(statement.to_string()); let CreateExternalTable { name, columns, @@ -480,6 +481,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { delimiter, table_partition_cols, if_not_exists, + definition, })) } From c8a5c807726211a027d5db6e338cc6ccc470c39b Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 3 Sep 2022 06:48:39 -0400 Subject: [PATCH 2/2] Use builder-style API to set ListingTable definition --- benchmarks/src/bin/h2o.rs | 2 +- benchmarks/src/bin/tpch.rs | 2 +- .../core/benches/sort_limit_query_sql.rs | 2 +- datafusion/core/src/catalog/schema.rs | 2 +- .../core/src/datasource/listing/table.rs | 25 +++++++++++-------- datafusion/core/src/execution/context.rs | 10 ++++---- datafusion/core/tests/path_partition.rs | 4 +-- datafusion/expr/src/logical_plan/plan.rs | 2 +- datafusion/proto/src/logical_plan.rs | 2 +- 9 files changed, 27 insertions(+), 24 deletions(-) diff --git a/benchmarks/src/bin/h2o.rs b/benchmarks/src/bin/h2o.rs index b0bff885baaa..88f4084e87a5 100644 --- a/benchmarks/src/bin/h2o.rs +++ b/benchmarks/src/bin/h2o.rs @@ -83,7 +83,7 @@ async fn group_by(opt: &GroupBy) -> Result<()> { let listing_config = ListingTableConfig::new(ListingTableUrl::parse(path)?) .with_listing_options(ListingOptions::new(Arc::new(CsvFormat::default()))) .with_schema(Arc::new(schema)); - let csv = ListingTable::try_new(listing_config, None)?; + let csv = ListingTable::try_new(listing_config)?; let partition_size = num_cpus::get(); let memtable = MemTable::load(Arc::new(csv), Some(partition_size), &ctx.state()).await?; diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs index a6ae9c61d1e1..08374bfcc5b2 100644 --- a/benchmarks/src/bin/tpch.rs +++ b/benchmarks/src/bin/tpch.rs @@ -429,7 +429,7 @@ fn get_table( .with_listing_options(options) .with_schema(schema); - Ok(Arc::new(ListingTable::try_new(config, None)?)) + Ok(Arc::new(ListingTable::try_new(config)?)) } fn get_schema(table: &str) -> Schema { diff --git a/datafusion/core/benches/sort_limit_query_sql.rs b/datafusion/core/benches/sort_limit_query_sql.rs index 95b4c6592669..e7aa33bd70bd 100644 --- a/datafusion/core/benches/sort_limit_query_sql.rs +++ b/datafusion/core/benches/sort_limit_query_sql.rs @@ -74,7 +74,7 @@ fn create_context() -> Arc> { .with_listing_options(listing_options) .with_schema(schema); - let csv = async { ListingTable::try_new(config, None).unwrap() }; + let csv = async { ListingTable::try_new(config).unwrap() }; let rt = Runtime::new().unwrap(); diff --git a/datafusion/core/src/catalog/schema.rs b/datafusion/core/src/catalog/schema.rs index 8ce064c74477..b886966bf6db 100644 --- a/datafusion/core/src/catalog/schema.rs +++ b/datafusion/core/src/catalog/schema.rs @@ -184,7 +184,7 @@ mod tests { .infer(&ctx.state()) .await .unwrap(); - let table = ListingTable::try_new(config, None).unwrap(); + let table = ListingTable::try_new(config).unwrap(); schema .register_table("alltypes_plain".to_string(), Arc::new(table)) diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index ee741a795515..e77773a822cd 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -48,7 +48,7 @@ use super::PartitionedFile; use super::helpers::{expr_applicable_for_cols, pruned_partition_list, split_files}; -/// Configuration for creating a 'ListingTable' +/// Configuration for creating a 'ListingTable' pub struct ListingTableConfig { /// Paths on the `ObjectStore` for creating `ListingTable`. /// They should share the same schema and object store. @@ -257,10 +257,7 @@ impl ListingTable { /// If the schema is provided then it must be resolved before creating the table /// and should contain the fields of the file without the table /// partitioning columns. - pub fn try_new( - config: ListingTableConfig, - definition: Option, - ) -> Result { + pub fn try_new(config: ListingTableConfig) -> Result { let file_schema = config .file_schema .ok_or_else(|| DataFusionError::Internal("No schema provided.".into()))?; @@ -284,12 +281,18 @@ impl ListingTable { file_schema, table_schema: Arc::new(Schema::new(table_fields)), options, - definition, + definition: None, }; Ok(table) } + /// Specify the SQL definition for this table, if any + pub fn with_definition(mut self, defintion: Option) -> Self { + self.definition = defintion; + self + } + /// Get paths ref pub fn table_paths(&self) -> &Vec { &self.table_paths @@ -469,7 +472,7 @@ mod tests { let config = ListingTableConfig::new(table_path) .with_listing_options(opt) .with_schema(schema); - let table = ListingTable::try_new(config, None)?; + let table = ListingTable::try_new(config)?; let exec = table.scan(&state, &None, &[], None).await?; assert_eq!(exec.statistics().num_rows, Some(8)); @@ -498,7 +501,7 @@ mod tests { let config = ListingTableConfig::new(table_path) .with_listing_options(opt) .with_schema(file_schema); - let table = ListingTable::try_new(config, None)?; + let table = ListingTable::try_new(config)?; assert_eq!( columns(&table.schema()), @@ -669,7 +672,7 @@ mod tests { let config = ListingTableConfig::new(table_path) .infer(&ctx.state()) .await?; - let table = ListingTable::try_new(config, None)?; + let table = ListingTable::try_new(config)?; Ok(Arc::new(table)) } @@ -701,7 +704,7 @@ mod tests { .with_listing_options(opt) .with_schema(Arc::new(schema)); - let table = ListingTable::try_new(config, None)?; + let table = ListingTable::try_new(config)?; let (file_list, _) = table.list_files_for_scan(&ctx.state(), &[], None).await?; @@ -741,7 +744,7 @@ mod tests { .with_listing_options(opt) .with_schema(Arc::new(schema)); - let table = ListingTable::try_new(config, None)?; + let table = ListingTable::try_new(config)?; let (file_list, _) = table.list_files_for_scan(&ctx.state(), &[], None).await?; diff --git a/datafusion/core/src/execution/context.rs b/datafusion/core/src/execution/context.rs index 7794a421b674..2a6e070dac3a 100644 --- a/datafusion/core/src/execution/context.rs +++ b/datafusion/core/src/execution/context.rs @@ -560,7 +560,7 @@ impl SessionContext { let config = ListingTableConfig::new(table_path) .with_listing_options(listing_options) .with_schema(resolved_schema); - let provider = ListingTable::try_new(config, None)?; + let provider = ListingTable::try_new(config)?; self.read_table(Arc::new(provider)) } @@ -586,7 +586,7 @@ impl SessionContext { let config = ListingTableConfig::new(table_path) .with_listing_options(listing_options) .with_schema(resolved_schema); - let provider = ListingTable::try_new(config, None)?; + let provider = ListingTable::try_new(config)?; self.read_table(Arc::new(provider)) } @@ -620,7 +620,7 @@ impl SessionContext { .with_listing_options(listing_options) .with_schema(resolved_schema); - let provider = ListingTable::try_new(config, None)?; + let provider = ListingTable::try_new(config)?; self.read_table(Arc::new(provider)) } @@ -644,7 +644,7 @@ impl SessionContext { .with_listing_options(listing_options) .with_schema(resolved_schema); - let provider = ListingTable::try_new(config, None)?; + let provider = ListingTable::try_new(config)?; self.read_table(Arc::new(provider)) } @@ -676,7 +676,7 @@ impl SessionContext { let config = ListingTableConfig::new(table_path) .with_listing_options(options) .with_schema(resolved_schema); - let table = ListingTable::try_new(config, sql)?; + let table = ListingTable::try_new(config)?.with_definition(sql); self.register_table(name, Arc::new(table))?; Ok(()) } diff --git a/datafusion/core/tests/path_partition.rs b/datafusion/core/tests/path_partition.rs index 4aee7b672410..fca9b9a43b1c 100644 --- a/datafusion/core/tests/path_partition.rs +++ b/datafusion/core/tests/path_partition.rs @@ -441,7 +441,7 @@ fn register_partitioned_aggregate_csv( let config = ListingTableConfig::new(table_path) .with_listing_options(options) .with_schema(file_schema); - let table = ListingTable::try_new(config, None).unwrap(); + let table = ListingTable::try_new(config).unwrap(); ctx.register_table("t", Arc::new(table)) .expect("registering listing table failed"); @@ -479,7 +479,7 @@ async fn register_partitioned_alltypes_parquet( .with_listing_options(options) .with_schema(file_schema); - let table = ListingTable::try_new(config, None).unwrap(); + let table = ListingTable::try_new(config).unwrap(); ctx.register_table("t", Arc::new(table)) .expect("registering listing table failed"); diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 6d7b2d962f68..b35790b46a05 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -1261,7 +1261,7 @@ pub struct CreateExternalTable { pub table_partition_cols: Vec, /// Option to not error if table already exists pub if_not_exists: bool, - /// SQL used to create the view, if available + /// SQL used to create the table, if available pub definition: Option, } diff --git a/datafusion/proto/src/logical_plan.rs b/datafusion/proto/src/logical_plan.rs index ef2725639d86..0e4146313b0d 100644 --- a/datafusion/proto/src/logical_plan.rs +++ b/datafusion/proto/src/logical_plan.rs @@ -430,7 +430,7 @@ impl AsLogicalPlan for LogicalPlanNode { .with_listing_options(options) .with_schema(Arc::new(schema)); - let provider = ListingTable::try_new(config, None)?; + let provider = ListingTable::try_new(config)?; LogicalPlanBuilder::scan_with_filters( &scan.table_name,