Skip to content

Commit

Permalink
fix: pass quote parameter to CSV writer (apache#10671)
Browse files Browse the repository at this point in the history
  • Loading branch information
DDtKey authored May 26, 2024
1 parent 0836500 commit c095fee
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 0 deletions.
1 change: 1 addition & 0 deletions datafusion/common/src/file_options/csv_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ impl TryFrom<&CsvOptions> for CsvWriterOptions {
fn try_from(value: &CsvOptions) -> Result<Self> {
let mut builder = WriterBuilder::default()
.with_header(value.has_header.unwrap_or(false))
.with_quote(value.quote)
.with_delimiter(value.delimiter);

if let Some(v) = &value.date_format {
Expand Down
41 changes: 41 additions & 0 deletions datafusion/sqllogictest/test_files/csv_files.slt
Original file line number Diff line number Diff line change
Expand Up @@ -161,3 +161,44 @@ physical_plan
01)SortPreservingMergeExec: [int_col@0 ASC NULLS LAST]
02)--SortExec: expr=[int_col@0 ASC NULLS LAST], preserve_partitioning=[true]
03)----CsvExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/csv_files/csv_partitions/1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/csv_files/csv_partitions/2.csv]]}, projection=[int_col, string_col, bigint_col, partition_col], has_header=false


# ensure that correct quote character is used when writing to csv
statement ok
CREATE TABLE table_with_necessary_quoting (
int_col INT,
string_col TEXT,
) AS VALUES
(1, 'e|e|e'),
(2, 'f|f|f'),
(3, 'g|g|g'),
(4, 'h|h|h');

# quote is required because `|` is delimiter and part of the data
query IT
COPY table_with_necessary_quoting TO 'test_files/scratch/csv_files/table_with_necessary_quoting.csv'
STORED AS csv
OPTIONS ('format.quote' '~',
'format.delimiter' '|',
'format.has_header' 'true');
----
4

# read the stored csv file with quote character
statement ok
CREATE EXTERNAL TABLE stored_table_with_necessary_quoting (
c1 VARCHAR,
c2 VARCHAR
) STORED AS CSV
LOCATION 'test_files/scratch/csv_files/table_with_necessary_quoting.csv'
OPTIONS ('format.quote' '~',
'format.delimiter' '|',
'format.has_header' 'true');

query TT
select * from stored_table_with_necessary_quoting;
----
1 e|e|e
2 f|f|f
3 g|g|g
4 h|h|h

0 comments on commit c095fee

Please sign in to comment.