Skip to content

Commit

Permalink
document describe_partition, add back repartition options to one of t…
Browse files Browse the repository at this point in the history
…he csv empty files tests
  • Loading branch information
Blizzara committed Dec 16, 2024
1 parent 38cdbf2 commit 3f58b02
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 1 deletion.
7 changes: 6 additions & 1 deletion datafusion/core/src/datasource/file_format/csv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1320,7 +1320,12 @@ mod tests {
/// (file is empty)
#[tokio::test]
async fn test_csv_multiple_empty_files() -> Result<()> {
let ctx = SessionContext::new();
// Testing that partitioning doesn't break with empty files
let config = SessionConfig::new()
.with_repartition_file_scans(true)
.with_repartition_file_min_size(0)
.with_target_partitions(4);
let ctx = SessionContext::new_with_config(config);
let file_format = Arc::new(CsvFormat::default().with_has_header(false));
let listing_options = ListingOptions::new(file_format.clone())
.with_file_extension(file_format.get_ext());
Expand Down
1 change: 1 addition & 0 deletions datafusion/core/src/datasource/listing/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,7 @@ mod tests {
);
}

/// Describe a partition as a (path, depth, files) tuple for easier assertions
fn describe_partition(partition: &Partition) -> (&str, usize, Vec<&str>) {
(
partition.path.as_ref(),
Expand Down

0 comments on commit 3f58b02

Please sign in to comment.