From 3f58b0270a89cdabd9062cf204cfb69687e76e48 Mon Sep 17 00:00:00 2001 From: Arttu Voutilainen Date: Mon, 16 Dec 2024 10:03:20 +0100 Subject: [PATCH] document describe_partition, add back repartition options to one of the csv empty files tests --- datafusion/core/src/datasource/file_format/csv.rs | 7 ++++++- datafusion/core/src/datasource/listing/helpers.rs | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 0278cd5e6514..e9a93475d3ce 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -1320,7 +1320,12 @@ mod tests { /// (file is empty) #[tokio::test] async fn test_csv_multiple_empty_files() -> Result<()> { - let ctx = SessionContext::new(); + // Testing that partitioning doesn't break with empty files + let config = SessionConfig::new() + .with_repartition_file_scans(true) + .with_repartition_file_min_size(0) + .with_target_partitions(4); + let ctx = SessionContext::new_with_config(config); let file_format = Arc::new(CsvFormat::default().with_has_header(false)); let listing_options = ListingOptions::new(file_format.clone()) .with_file_extension(file_format.get_ext()); diff --git a/datafusion/core/src/datasource/listing/helpers.rs b/datafusion/core/src/datasource/listing/helpers.rs index c8379a643148..228b9a4e9f6b 100644 --- a/datafusion/core/src/datasource/listing/helpers.rs +++ b/datafusion/core/src/datasource/listing/helpers.rs @@ -680,6 +680,7 @@ mod tests { ); } + /// Describe a partition as a (path, depth, files) tuple for easier assertions fn describe_partition(partition: &Partition) -> (&str, usize, Vec<&str>) { ( partition.path.as_ref(),