ensure dates are parsed appropriately

maxcountryman · Jun 21, 2022 · 0719d65 · 0719d65
1 parent 4aeddce
commit 0719d65
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 12 deletions.
diff --git a/src/main.rs b/src/main.rs
@@ -95,6 +95,7 @@ fn main() -> Result<(), Error> {
     let props = Some(
         WriterProperties::builder()
             .set_compression(args.compression.into())
+            .set_created_by(String::from("warc-parquet"))
             .build(),
     );
     let mut writer = ArrowWriter::try_new(parquet_file, batch.schema(), props)?;

diff --git a/src/record_columns.rs b/src/record_columns.rs
@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use arrow::array::{ArrayRef, BinaryArray, StringArray, TimestampSecondArray, UInt32Array};
+use arrow::array::{ArrayRef, BinaryArray, StringArray, TimestampMillisecondArray, UInt32Array};
 use chrono::NaiveDateTime;
 use warc::{BufferedBody, Record, WarcHeader};
 
@@ -71,16 +71,19 @@ impl RecordColumns {
             .unwrap()]))
     }
 
-    fn date(&self) -> Arc<TimestampSecondArray> {
-        Arc::new(TimestampSecondArray::from(vec![self
-            .record
-            .header(WarcHeader::Date)
-            .map(|s| {
-                NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%SZ")
-                    .unwrap()
-                    .timestamp()
-            })
-            .unwrap()]))
+    fn date(&self) -> Arc<TimestampMillisecondArray> {
+        Arc::new(TimestampMillisecondArray::from_vec(
+            vec![self
+                .record
+                .header(WarcHeader::Date)
+                .map(|s| {
+                    NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%SZ")
+                        .unwrap()
+                        .timestamp_millis()
+                })
+                .unwrap()],
+            None,
+        ))
     }
 
     fn r#type(&self) -> Arc<StringArray> {

diff --git a/src/schema.rs b/src/schema.rs
@@ -7,7 +7,11 @@ fn fields() -> Vec<Field> {
         // Mandatory fields.
         Field::new("id", DataType::Utf8, false),
         Field::new("content_length", DataType::UInt32, false),
-        Field::new("date", DataType::Timestamp(TimeUnit::Second, None), false),
+        Field::new(
+            "date",
+            DataType::Timestamp(TimeUnit::Millisecond, None),
+            false,
+        ),
         Field::new("type", DataType::Utf8, false),
         // Optional fields.
         Field::new("content_type", DataType::Utf8, true),