Merge pull request duckdb#12029 from pdet/fix_time_timestamp_csv

Adding Fix to properly pass timestamp/date formats in the relational API for CSV Files
kryonix · May 14, 2024 · 5bd8716 · 5bd8716
2 parents d4c6e67 + de9b8df
commit 5bd8716
Show file tree

Hide file tree

Showing 4 changed files with 384 additions and 6 deletions.
diff --git a/data/csv/weather.csv b/data/csv/weather.csv
@@ -0,0 +1,367 @@
+date,maximum_temperature,minimum_temperature,average_temperature,precipitation,snow_fall,snow_depth
+1-1-2016,42,34,38.0,0.00,0.0,0
+2-1-2016,40,32,36.0,0.00,0.0,0
+3-1-2016,45,35,40.0,0.00,0.0,0
+4-1-2016,36,14,25.0,0.00,0.0,0
+5-1-2016,29,11,20.0,0.00,0.0,0
+6-1-2016,41,25,33.0,0.00,0.0,0
+7-1-2016,46,31,38.5,0.00,0.0,0
+8-1-2016,46,31,38.5,0.00,0.0,0
+9-1-2016,47,40,43.5,T,0.0,0
+10-1-2016,59,40,49.5,1.80,0.0,0
+11-1-2016,40,26,33.0,0.00,0.0,0
+12-1-2016,44,25,34.5,0.00,T,0
+13-1-2016,30,22,26.0,0.00,0.0,0
+14-1-2016,38,22,30.0,0.00,T,0
+15-1-2016,51,34,42.5,T,0.0,0
+16-1-2016,52,42,47.0,0.24,0.0,0
+17-1-2016,42,30,36.0,0.05,0.4,0
+18-1-2016,31,18,24.5,T,T,T
+19-1-2016,28,16,22.0,0.00,0.0,T
+20-1-2016,37,27,32.0,0.00,0.0,T
+21-1-2016,36,26,31.0,0.00,0.0,0
+22-1-2016,30,21,25.5,0.01,0.2,0
+23-1-2016,27,24,25.5,2.31,27.3,6
+24-1-2016,35,20,27.5,T,T,22
+25-1-2016,39,28,33.5,0.00,0.0,19
+26-1-2016,48,38,43.0,0.00,0.0,17
+27-1-2016,47,34,40.5,T,0.0,9
+28-1-2016,42,32,37.0,0.00,0.0,6
+29-1-2016,41,30,35.5,0.00,0.0,6
+30-1-2016,39,28,33.5,0.00,0.0,6
+31-1-2016,56,36,46.0,0.00,0.0,4
+1-2-2016,59,44,51.5,0.01,0.0,2
+2-2-2016,50,38,44.0,0.00,0.0,T
+3-2-2016,59,42,50.5,0.73,0.0,0
+4-2-2016,59,44,51.5,T,0.0,0
+5-2-2016,44,31,37.5,0.53,2.5,1
+6-2-2016,40,30,35.0,0.00,0.0,0
+7-2-2016,47,33,40.0,0.00,0.0,0
+8-2-2016,39,28,33.5,0.05,0.1,0
+9-2-2016,36,27,31.5,0.00,T,T
+10-2-2016,39,31,35.0,0.01,T,0
+11-2-2016,31,18,24.5,T,T,0
+12-2-2016,27,15,21.0,0.00,0.0,0
+13-2-2016,22,6,14.0,0.00,0.0,0
+14-2-2016,15,-1,7.0,0.00,0.0,0
+15-2-2016,35,13,24.0,0.44,1.4,0
+16-2-2016,54,35,44.5,1.01,0.0,0
+17-2-2016,39,35,37.0,0.00,0.0,0
+18-2-2016,36,27,31.5,0.00,0.0,0
+19-2-2016,39,24,31.5,0.00,0.0,0
+20-2-2016,61,39,50.0,0.00,0.0,0
+21-2-2016,55,44,49.5,0.03,0.0,0
+22-2-2016,52,38,45.0,0.00,0.0,0
+23-2-2016,40,35,37.5,0.30,T,0
+24-2-2016,60,36,48.0,1.22,0.0,0
+25-2-2016,61,37,49.0,0.02,0.0,0
+26-2-2016,39,27,33.0,0.00,0.0,0
+27-2-2016,41,26,33.5,0.00,0.0,0
+28-2-2016,60,38,49.0,0.00,0.0,0
+29-2-2016,61,47,54.0,0.05,0.0,0
+1-3-2016,52,39,45.5,0.00,0.0,0
+2-3-2016,55,29,42.0,0.14,0.0,0
+3-3-2016,36,26,31.0,0.00,0.0,0
+4-3-2016,39,30,34.5,0.11,0.4,T
+5-3-2016,41,28,34.5,0.00,0.0,0
+6-3-2016,44,32,38.0,0.00,0.0,0
+7-3-2016,60,36,48.0,0.00,0.0,0
+8-3-2016,67,47,57.0,0.00,0.0,0
+9-3-2016,77,44,60.5,0.00,0.0,0
+10-3-2016,79,63,71.0,0.00,0.0,0
+11-3-2016,68,48,58.0,0.06,0.0,0
+12-3-2016,59,40,49.5,0.00,0.0,0
+13-3-2016,62,50,56.0,T,0.0,0
+14-3-2016,51,40,45.5,0.29,0.0,0
+15-3-2016,57,44,50.5,0.00,0.0,0
+16-3-2016,65,48,56.5,0.02,0.0,0
+17-3-2016,63,45,54.0,T,0.0,0
+18-3-2016,57,42,49.5,0.00,0.0,0
+19-3-2016,46,36,41.0,0.00,0.0,0
+20-3-2016,43,32,37.5,0.07,T,0
+21-3-2016,50,32,41.0,0.06,0.5,T
+22-3-2016,56,35,45.5,0.00,0.0,0
+23-3-2016,71,48,59.5,0.00,0.0,0
+24-3-2016,55,44,49.5,0.00,0.0,0
+25-3-2016,72,44,58.0,0.04,0.0,0
+26-3-2016,55,38,46.5,0.00,0.0,0
+27-3-2016,55,43,49.0,0.00,0.0,0
+28-3-2016,62,42,52.0,0.38,0.0,0
+29-3-2016,53,40,46.5,0.00,0.0,0
+30-3-2016,56,37,46.5,0.00,0.0,0
+31-3-2016,73,49,61.0,0.00,0.0,0
+1-4-2016,79,61,70.0,0.02,0.0,0
+2-4-2016,61,49,55.0,0.16,0.0,0
+3-4-2016,50,34,42.0,0.09,T,0
+4-4-2016,45,29,37.0,0.47,T,0
+5-4-2016,43,26,34.5,0.00,0.0,0
+6-4-2016,48,30,39.0,0.00,0.0,0
+7-4-2016,58,48,53.0,0.09,0.0,0
+8-4-2016,50,40,45.0,0.01,0.0,0
+9-4-2016,43,36,39.5,0.11,T,0
+10-4-2016,50,31,40.5,0.00,0.0,0
+11-4-2016,65,43,54.0,0.01,0.0,0
+12-4-2016,59,45,52.0,0.20,0.0,0
+13-4-2016,58,40,49.0,0.00,0.0,0
+14-4-2016,62,43,52.5,0.00,0.0,0
+15-4-2016,65,42,53.5,0.00,0.0,0
+16-4-2016,68,43,55.5,0.00,0.0,0
+17-4-2016,75,44,59.5,0.00,0.0,0
+18-4-2016,82,51,66.5,0.00,0.0,0
+19-4-2016,73,55,64.0,0.00,0.0,0
+20-4-2016,69,49,59.0,0.00,0.0,0
+21-4-2016,73,49,61.0,0.00,0.0,0
+22-4-2016,79,62,70.5,T,0.0,0
+23-4-2016,71,54,62.5,0.16,0.0,0
+24-4-2016,68,47,57.5,0.00,0.0,0
+25-4-2016,69,50,59.5,0.00,0.0,0
+26-4-2016,60,47,53.5,0.24,0.0,0
+27-4-2016,62,46,54.0,0.00,0.0,0
+28-4-2016,59,48,53.5,0.00,0.0,0
+29-4-2016,58,45,51.5,0.05,0.0,0
+30-4-2016,65,46,55.5,0.00,0.0,0
+1-5-2016,51,45,48.0,0.16,0.0,0
+2-5-2016,60,45,52.5,0.04,0.0,0
+3-5-2016,56,51,53.5,0.61,0.0,0
+4-5-2016,52,48,50.0,0.01,0.0,0
+5-5-2016,57,46,51.5,0.00,0.0,0
+6-5-2016,54,48,51.0,0.54,0.0,0
+7-5-2016,60,48,54.0,0.00,0.0,0
+8-5-2016,66,49,57.5,0.16,0.0,0
+9-5-2016,72,52,62.0,0.00,0.0,0
+10-5-2016,63,50,56.5,0.00,0.0,0
+11-5-2016,76,50,63.0,0.00,0.0,0
+12-5-2016,80,56,68.0,0.00,0.0,0
+13-5-2016,65,57,61.0,0.25,0.0,0
+14-5-2016,73,56,64.5,0.00,0.0,0
+15-5-2016,59,46,52.5,0.00,0.0,0
+16-5-2016,66,43,54.5,0.00,0.0,0
+17-5-2016,64,52,58.0,0.00,0.0,0
+18-5-2016,68,52,60.0,T,0.0,0
+19-5-2016,73,54,63.5,0.00,0.0,0
+20-5-2016,76,53,64.5,0.00,0.0,0
+21-5-2016,66,54,60.0,0.04,0.0,0
+22-5-2016,70,52,61.0,0.09,0.0,0
+23-5-2016,78,56,67.0,0.02,0.0,0
+24-5-2016,73,58,65.5,0.18,0.0,0
+25-5-2016,88,61,74.5,0.00,0.0,0
+26-5-2016,90,69,79.5,0.00,0.0,0
+27-5-2016,87,73,80.0,0.00,0.0,0
+28-5-2016,92,71,81.5,0.00,0.0,0
+29-5-2016,87,70,78.5,T,0.0,0
+30-5-2016,82,68,75.0,1.65,0.0,0
+31-5-2016,85,71,78.0,0.00,0.0,0
+1-6-2016,83,66,74.5,0.00,0.0,0
+2-6-2016,78,62,70.0,0.00,0.0,0
+3-6-2016,70,63,66.5,0.04,0.0,0
+4-6-2016,83,66,74.5,0.40,0.0,0
+5-6-2016,71,65,68.0,0.91,0.0,0
+6-6-2016,83,65,74.0,0.00,0.0,0
+7-6-2016,85,64,74.5,T,0.0,0
+8-6-2016,67,52,59.5,0.45,0.0,0
+9-6-2016,71,54,62.5,0.00,0.0,0
+10-6-2016,77,57,67.0,0.00,0.0,0
+11-6-2016,88,59,73.5,0.00,0.0,0
+12-6-2016,83,62,72.5,0.00,0.0,0
+13-6-2016,74,57,65.5,0.00,0.0,0
+14-6-2016,79,58,68.5,0.00,0.0,0
+15-6-2016,85,62,73.5,0.00,0.0,0
+16-6-2016,74,65,69.5,0.22,0.0,0
+17-6-2016,78,63,70.5,0.00,0.0,0
+18-6-2016,87,61,74.0,0.00,0.0,0
+19-6-2016,88,66,77.0,0.00,0.0,0
+20-6-2016,84,64,74.0,0.00,0.0,0
+21-6-2016,87,72,79.5,T,0.0,0
+22-6-2016,86,68,77.0,0.00,0.0,0
+23-6-2016,83,69,76.0,0.00,0.0,0
+24-6-2016,84,67,75.5,0.00,0.0,0
+25-6-2016,86,64,75.0,0.00,0.0,0
+26-6-2016,87,67,77.0,0.00,0.0,0
+27-6-2016,83,67,75.0,0.45,0.0,0
+28-6-2016,76,68,72.0,0.12,0.0,0
+29-6-2016,83,67,75.0,0.01,0.0,0
+30-6-2016,85,67,76.0,0.00,0.0,0
+1-7-2016,79,66,72.5,0.83,0,0
+2-7-2016,76,63,69.5,0,0,0
+3-7-2016,78,64,71,0,0,0
+4-7-2016,84,66,75,0.49,0,0
+5-7-2016,86,69,77.5,0.66,0,0
+6-7-2016,91,75,83,0,0,0
+7-7-2016,89,77,83,0.04,0,0
+8-7-2016,86,67,76.5,0.08,0,0
+9-7-2016,72,65,68.5,0.53,0,0
+10-7-2016,80,66,73,T,0,0
+11-7-2016,81,65,73,0,0,0
+12-7-2016,82,68,75,0,0,0
+13-7-2016,85,71,78,0,0,0
+14-7-2016,88,73,80.5,0.62,0,0
+15-7-2016,88,79,83.5,0,0,0
+16-7-2016,90,75,82.5,T,0,0
+17-7-2016,89,75,82,0,0,0
+18-7-2016,93,72,82.5,0.35,0,0
+19-7-2016,83,73,78,0,0,0
+20-7-2016,85,68,76.5,0,0,0
+21-7-2016,90,71,80.5,0,0,0
+22-7-2016,94,74,84,0,0,0
+23-7-2016,96,80,88,0,0,0
+24-7-2016,94,75,84.5,0,0,0
+25-7-2016,93,73,83,1,0,0
+26-7-2016,89,72,80.5,0,0,0
+27-7-2016,91,74,82.5,0,0,0
+28-7-2016,95,75,85,T,0,0
+29-7-2016,85,69,77,1.09,0,0
+30-7-2016,84,73,78.5,0.25,0,0
+31-7-2016,78,71,74.5,1.08,0,0
+1-8-2016,80,69,74.5,T,0,0
+2-8-2016,79,68,73.5,0,0,0
+3-8-2016,80,66,73,0,0,0
+4-8-2016,81,67,74,0,0,0
+5-8-2016,83,69,76,0,0,0
+6-8-2016,87,72,79.5,0.05,0,0
+7-8-2016,86,70,78,0,0,0
+8-8-2016,86,71,78.5,0,0,0
+9-8-2016,87,71,79,0,0,0
+10-8-2016,86,75,80.5,0.09,0,0
+11-8-2016,91,74,82.5,0.15,0,0
+12-8-2016,93,73,83,0.32,0,0
+13-8-2016,96,81,88.5,0,0,0
+14-8-2016,94,78,86,0.06,0,0
+15-8-2016,92,77,84.5,0,0,0
+16-8-2016,87,78,82.5,0.11,0,0
+17-8-2016,85,77,81,0.01,0,0
+18-8-2016,85,72,78.5,0.03,0,0
+19-8-2016,88,74,81,0.01,0,0
+20-8-2016,83,70,76.5,0.82,0,0
+21-8-2016,86,73,79.5,0.31,0,0
+22-8-2016,79,65,72,0,0,0
+23-8-2016,82,61,71.5,0,0,0
+24-8-2016,88,68,78,0,0,0
+25-8-2016,86,69,77.5,0.01,0,0
+26-8-2016,90,75,82.5,0,0,0
+27-8-2016,89,73,81,0,0,0
+28-8-2016,89,72,80.5,0,0,0
+29-8-2016,91,74,82.5,0,0,0
+30-8-2016,86,69,77.5,0,0,0
+31-8-2016,89,74,81.5,T,0,0
+1-9-2016,79,69,74,0.5,0,0
+2-9-2016,81,66,73.5,0,0,0
+3-9-2016,75,66,70.5,0,0,0
+4-9-2016,80,65,72.5,0,0,0
+5-9-2016,84,65,74.5,0,0,0
+6-9-2016,80,71,75.5,T,0,0
+7-9-2016,85,71,78,0,0,0
+8-9-2016,89,71,80,0,0,0
+9-9-2016,91,75,83,0.22,0,0
+10-9-2016,90,74,82,T,0,0
+11-9-2016,83,67,75,0,0,0
+12-9-2016,78,62,70,0,0,0
+13-9-2016,83,64,73.5,0,0,0
+14-9-2016,91,67,79,0.56,0,0
+15-9-2016,74,61,67.5,0,0,0
+16-9-2016,73,61,67,0,0,0
+17-9-2016,77,59,68,0,0,0
+18-9-2016,82,70,76,0,0,0
+19-9-2016,76,69,72.5,0.68,0,0
+20-9-2016,82,68,75,0,0,0
+21-9-2016,84,68,76,0,0,0
+22-9-2016,86,65,75.5,0,0,0
+23-9-2016,87,64,75.5,0.01,0,0
+24-9-2016,71,58,64.5,0.2,0,0
+25-9-2016,70,54,62,0,0,0
+26-9-2016,74,54,64,0,0,0
+27-9-2016,74,64,69,0.22,0,0
+28-9-2016,69,56,62.5,0,0,0
+29-9-2016,64,57,60.5,0,0,0
+30-9-2016,59,56,57.5,0.4,0,0
+1-10-2016,62,56,59,0,0,0
+2-10-2016,63,57,60,0,0,0
+3-10-2016,72,60,66,0,0,0
+4-10-2016,69,60,64.5,0,0,0
+5-10-2016,67,53,60,0,0,0
+6-10-2016,73,55,64,0,0,0
+7-10-2016,75,57,66,0,0,0
+8-10-2016,68,58,63,0.23,0,0
+9-10-2016,65,51,58,0.55,0,0
+10-10-2016,64,47,55.5,0,0,0
+11-10-2016,63,46,54.5,0,0,0
+12-10-2016,66,53,59.5,0,0,0
+13-10-2016,67,53,60,T,0,0
+14-10-2016,62,47,54.5,0,0,0
+15-10-2016,65,45,55,0,0,0
+16-10-2016,69,53,61,0,0,0
+17-10-2016,81,63,72,0,0,0
+18-10-2016,81,67,74,0,0,0
+19-10-2016,85,65,75,0,0,0
+20-10-2016,70,62,66,0,0,0
+21-10-2016,69,57,63,1.11,0,0
+22-10-2016,57,47,52,0.29,0,0
+23-10-2016,61,45,53,0,0,0
+24-10-2016,62,47,54.5,T,0,0
+25-10-2016,52,43,47.5,0,0,0
+26-10-2016,51,38,44.5,0,0,0
+27-10-2016,55,40,47.5,1.41,0,0
+28-10-2016,51,42,46.5,0,0,0
+29-10-2016,64,39,51.5,0,0,0
+30-10-2016,76,54,65,0.56,0,0
+31-10-2016,54,44,49,0,0,0
+1-11-2016,58,40,49,0,0,0
+2-11-2016,70,54,62,0,0,0
+3-11-2016,72,57,64.5,0,0,0
+4-11-2016,61,47,54,0,0,0
+5-11-2016,62,44,53,0,0,0
+6-11-2016,59,45,52,0,0,0
+7-11-2016,53,41,47,0,0,0
+8-11-2016,66,41,53.5,0,0,0
+9-11-2016,59,49,54,0.06,0,0
+10-11-2016,56,40,48,0,0,0
+11-11-2016,63,41,52,0,0,0
+12-11-2016,50,37,43.5,0,0,0
+13-11-2016,61,41,51,0,0,0
+14-11-2016,62,46,54,0,0,0
+15-11-2016,56,47,51.5,1.81,0,0
+16-11-2016,61,45,53,0,0,0
+17-11-2016,61,49,55,0,0,0
+18-11-2016,64,44,54,0,0,0
+19-11-2016,63,37,50,0.25,0,0
+20-11-2016,42,34,38,0.31,T,0
+21-11-2016,41,36,38.5,0,0,0
+22-11-2016,41,37,39,0,0,0
+23-11-2016,45,35,40,0,0,0
+24-11-2016,48,38,43,0.03,0,0
+25-11-2016,54,45,49.5,0.02,0,0
+26-11-2016,50,40,45,0,0,0
+27-11-2016,50,39,44.5,0,0,0
+28-11-2016,52,38,45,0,0,0
+29-11-2016,60,51,55.5,2.2,0,0
+30-11-2016,58,50,54,0.73,0,0
+1-12-2016,54,42,48,0.07,0,0
+2-12-2016,51,40,45.5,0,0,0
+3-12-2016,47,41,44,T,0,0
+4-12-2016,47,39,43,0,0,0
+5-12-2016,49,38,43.5,0.19,0,0
+6-12-2016,46,37,41.5,0.35,0,0
+7-12-2016,46,40,43,0.09,0,0
+8-12-2016,45,35,40,0,0,0
+9-12-2016,39,29,34,0,0,0
+10-12-2016,35,28,31.5,0,0,0
+11-12-2016,35,28,31.5,0.03,0.4,0
+12-12-2016,46,34,40,0.5,0,0
+13-12-2016,43,35,39,0,0,0
+14-12-2016,42,34,38,0,0,0
+15-12-2016,34,19,26.5,0,T,0
+16-12-2016,27,17,22,0,0,0
+17-12-2016,39,24,31.5,0.73,2.8,2
+18-12-2016,58,31,44.5,0.04,0,1
+19-12-2016,31,23,27,0,0,0
+20-12-2016,33,20,26.5,0,0,0
+21-12-2016,40,30,35,0,0,0
+22-12-2016,49,37,43,0,0,0
+23-12-2016,47,38,42.5,0,0,0
+24-12-2016,47,38,42.5,0.47,0,0
+25-12-2016,50,36,43,0,0,0
+26-12-2016,50,33,41.5,0.02,0,0
+27-12-2016,60,40,50,0,0,0
+28-12-2016,40,34,37,0,0,0
+29-12-2016,46,33,39.5,0.39,0,0
+30-12-2016,40,33,36.5,0.01,T,0
+31-12-2016,44,31,37.5,0,0,0
diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp
@@ -592,11 +592,13 @@ void CSVReaderOptions::ToNamedParameters(named_parameter_map_t &named_params) {
 	}
 	named_params["null_padding"] = Value::BOOLEAN(null_padding);
 	named_params["parallel"] = Value::BOOLEAN(parallel);
-	if (!date_format.at(LogicalType::DATE).format_specifier.empty()) {
-		named_params["dateformat"] = Value(date_format.at(LogicalType::DATE).format_specifier);
+	if (!dialect_options.date_format.at(LogicalType::DATE).GetValue().format_specifier.empty()) {
+		named_params["dateformat"] =
+		    Value(dialect_options.date_format.at(LogicalType::DATE).GetValue().format_specifier);
 	}
-	if (!date_format.at(LogicalType::TIMESTAMP).format_specifier.empty()) {
-		named_params["timestampformat"] = Value(date_format.at(LogicalType::TIMESTAMP).format_specifier);
+	if (!dialect_options.date_format.at(LogicalType::TIMESTAMP).GetValue().format_specifier.empty()) {
+		named_params["timestampformat"] =
+		    Value(dialect_options.date_format.at(LogicalType::TIMESTAMP).GetValue().format_specifier);
 	}
 
 	named_params["normalize_names"] = Value::BOOLEAN(normalize_names);

diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp
@@ -120,8 +120,6 @@ struct CSVReaderOptions {
 	string suffix;
 	string write_newline;
 
-	//! The date format to use (if any is specified)
-	map<LogicalTypeId, StrpTimeFormat> date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
 	//! The date format to use for writing (if any is specified)
 	map<LogicalTypeId, Value> write_date_format = {{LogicalTypeId::DATE, Value()}, {LogicalTypeId::TIMESTAMP, Value()}};
 	//! Whether or not a type format is specified