Utilise struct stats when available (#656)

* Log stats * Point unittest to problematic data * Add minimal test data to reproduce * Update test data * Fix test data * Add test * Update rust logging * It actually compiles! * I don't understand rust * Debug logging * Fix sample data * Tidy * Update rust/src/action.rs Co-authored-by: Will Jones <[email protected]> * Test data with more complex types * Still return json stats if there is an error parsing parquet stats * Better error message * Unittest covering more complex types * Support parsing structs * Compare struct comes out the same as json * Correct timestamp formatting * In progress better test and support for more columns * All types except decimal work * Test data with nested structs * Update test * All workng except decimal * Working decimal conversion * Update test data again * Passing test * Tidy * Tidy * Tidy * Remove .crc files * Remove unneeded return statements * Remove python test * Use from and reference instead of clone Co-authored-by: Will Jones <[email protected]> * Use into Co-authored-by: Will Jones <[email protected]> * dereferance timestamp Co-authored-by: Will Jones <[email protected]> * use into Co-authored-by: Will Jones <[email protected]> * Use reference to field Co-authored-by: Will Jones <[email protected]> * de-reference date Co-authored-by: Will Jones <[email protected]> * Update rust/tests/read_delta_test.rs Co-authored-by: Will Jones <[email protected]> Co-authored-by: Will Jones <[email protected]>
delta-io · Jul 6, 2022 · 8d7f3b6 · 8d7f3b6
1 parent 9bfa8ab
commit 8d7f3b6
Show file tree

Hide file tree

Showing 31 changed files with 144 additions and 9 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -24,6 +24,8 @@ chrono = "0"
 uuid = { version = "1", features = ["serde", "v4"] }
 lazy_static = "1"
 percent-encoding = "2"
+num-bigint = "0.4"
+num-traits = "0.2.15"
 
 # HTTP Client
 reqwest = { version = "0.11", default-features = false, features = [

diff --git a/rust/src/action.rs b/rust/src/action.rs
@@ -3,10 +3,13 @@
 #![allow(non_camel_case_types)]
 
 use crate::{schema::*, DeltaTableMetaData};
-use parquet::record::{ListAccessor, MapAccessor, RowAccessor};
+use chrono::{SecondsFormat, TimeZone, Utc};
+use num_bigint::BigInt;
+use num_traits::cast::ToPrimitive;
+use parquet::record::{Field, ListAccessor, MapAccessor, RowAccessor};
 use percent_encoding::percent_decode;
 use serde::{Deserialize, Serialize};
-use serde_json::{Map, Value};
+use serde_json::{json, Map, Value};
 use std::borrow::Borrow;
 use std::collections::HashMap;
 use std::hash::{Hash, Hasher};
@@ -114,7 +117,7 @@ impl ColumnCountStat {
 }
 
 /// Statistics associated with Add actions contained in the Delta log.
-#[derive(Serialize, Deserialize, Debug, Default)]
+#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
 pub struct Stats {
     /// Number of records in the file associated with the log action.
@@ -288,19 +291,34 @@ impl Add {
         decode_path(&self.path).map(|path| Self { path, ..self })
     }
 
+    /// Get whatever stats are available. Uses (parquet struct) parsed_stats if present falling back to json stats.
+    pub fn get_stats(&self) -> Result<Option<Stats>, serde_json::error::Error> {
+        match self.get_stats_parsed() {
+            Ok(Some(stats)) => Ok(Some(stats)),
+            Ok(None) => self.get_json_stats(),
+            Err(e) => {
+                log::error!(
+                    "Error when reading parquet stats {:?} {e}. Attempting to read json stats",
+                    self.stats_parsed
+                );
+                self.get_json_stats()
+            }
+        }
+    }
+
     /// Returns the serde_json representation of stats contained in the action if present.
     /// Since stats are defined as optional in the protocol, this may be None.
-    pub fn get_stats(&self) -> Result<Option<Stats>, serde_json::error::Error> {
+    pub fn get_json_stats(&self) -> Result<Option<Stats>, serde_json::error::Error> {
         self.stats
             .as_ref()
             .map_or(Ok(None), |s| serde_json::from_str(s))
     }
 
     /// Returns the composite HashMap representation of stats contained in the action if present.
     /// Since stats are defined as optional in the protocol, this may be None.
-    pub fn get_stats_parsed(&self) -> Result<Option<StatsParsed>, parquet::errors::ParquetError> {
+    pub fn get_stats_parsed(&self) -> Result<Option<Stats>, parquet::errors::ParquetError> {
         self.stats_parsed.as_ref().map_or(Ok(None), |record| {
-            let mut stats = StatsParsed::default();
+            let mut stats = Stats::default();
 
             for (i, (name, _)) in record.get_column_iter().enumerate() {
                 match name.as_str() {
@@ -315,7 +333,7 @@ impl Add {
                     "minValues" => match record.get_group(i) {
                         Ok(row) => {
                             for (name, field) in row.get_column_iter() {
-                                stats.min_values.insert(name.clone(), field.clone());
+                                stats.min_values.insert(name.clone(), field.into());
                             }
                         }
                         _ => {
@@ -325,7 +343,7 @@ impl Add {
                     "maxValues" => match record.get_group(i) {
                         Ok(row) => {
                             for (name, field) in row.get_column_iter() {
-                                stats.max_values.insert(name.clone(), field.clone());
+                                stats.max_values.insert(name.clone(), field.into());
                             }
                         }
                         _ => {
@@ -337,7 +355,7 @@ impl Add {
                             for (i, (name, _)) in row.get_column_iter().enumerate() {
                                 match row.get_long(i) {
                                     Ok(v) => {
-                                        stats.null_count.insert(name.clone(), v);
+                                        stats.null_count.insert(name.clone(), ColumnCountStat::Value(v));
                                     }
                                     _ => {
                                         log::error!("Expect type of stats_parsed.nullRecords value to be struct, got: {}", row);
@@ -364,6 +382,56 @@ impl Add {
     }
 }
 
+impl From<&Field> for ColumnValueStat {
+    fn from(field: &Field) -> Self {
+        match field {
+            Field::Group(group) => ColumnValueStat::Column(HashMap::from_iter(
+                group
+                    .get_column_iter()
+                    .map(|(field_name, field)| (field_name.clone(), field.into())),
+            )),
+            _ => ColumnValueStat::Value(primitive_parquet_field_to_json_value(field)),
+        }
+    }
+}
+
+fn primitive_parquet_field_to_json_value(field: &Field) -> serde_json::Value {
+    match field {
+        Field::Null => serde_json::Value::Null,
+        Field::Bool(value) => json!(value),
+        Field::Byte(value) => json!(value),
+        Field::Short(value) => json!(value),
+        Field::Int(value) => json!(value),
+        Field::Long(value) => json!(value),
+        Field::Float(value) => json!(value),
+        Field::Double(value) => json!(value),
+        Field::Str(value) => json!(value),
+        Field::Decimal(decimal) => match BigInt::from_signed_bytes_be(decimal.data()).to_f64() {
+            Some(int) => json!(int / (10_i64.pow((decimal.scale()).try_into().unwrap()) as f64)),
+            _ => serde_json::Value::Null,
+        },
+        Field::TimestampMillis(timestamp) => {
+            serde_json::Value::String(convert_timestamp_millis_to_string(*timestamp))
+        }
+        Field::Date(date) => serde_json::Value::String(convert_date_to_string(*date)),
+        _ => {
+            log::warn!("Unexpected field type {:?}", field,);
+            serde_json::Value::Null
+        }
+    }
+}
+
+fn convert_timestamp_millis_to_string(value: u64) -> String {
+    let dt = Utc.timestamp((value / 1000) as i64, ((value % 1000) * 1000000) as u32);
+    dt.to_rfc3339_opts(SecondsFormat::Millis, true)
+}
+
+fn convert_date_to_string(value: u32) -> String {
+    static NUM_SECONDS_IN_DAY: i64 = 60 * 60 * 24;
+    let dt = Utc.timestamp(value as i64 * NUM_SECONDS_IN_DAY, 0).date();
+    format!("{}", dt.format("%Y-%m-%d"))
+}
+
 /// Describes the data format of files in the table.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 pub struct Format {

diff --git a/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000000.json b/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000000.json
@@ -0,0 +1,3 @@
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"metaData":{"id":"9c4df48c-6085-4dcf-b73e-13147a5a405e","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"integer\",\"type\":\"integer\",\"nullable\":false,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpoint.writeStatsAsJson":"false","delta.checkpoint.writeStatsAsStruct":"true"},"createdTime":1656252116073}}
+{"commitInfo":{"timestamp":1656252116149,"userId":"6114986638742036","userName":"[email protected]","operation":"CREATE OR REPLACE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{\"delta.checkpoint.writeStatsAsJson\":\"false\",\"delta.checkpoint.writeStatsAsStruct\":\"true\"}"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"af1b716b-5ec8-41f6-9cc2-bb89e010f943"}}
diff --git a/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000001.json b/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000001.json
@@ -0,0 +1,3 @@
+{"metaData":{"id":"9c4df48c-6085-4dcf-b73e-13147a5a405e","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"integer\",\"type\":\"integer\",\"nullable\":false,\"metadata\":{}},{\"name\":\"null\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal\",\"type\":\"decimal(8,5)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"struct_element\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"string\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array\",\"type\":{\"type\":\"array\",\"elementType\":\"string\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"struct_element\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"nested_struct_element\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_map\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"struct_element\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"string\",\"valueContainsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpoint.writeStatsAsJson":"false","delta.checkpoint.writeStatsAsStruct":"true"},"createdTime":1656252116073}}
+{"add":{"path":"part-00000-51653f4d-b029-44bd-9fda-578e73518a26-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252122000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":0,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:01.678Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":0,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:01.678Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252122000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
+{"commitInfo":{"timestamp":1656252122901,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"4398d2b8-9a41-46ea-b92c-38820595bfec"}}
diff --git a/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000002.json b/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000002.json
@@ -0,0 +1,2 @@
+{"add":{"path":"part-00000-a222e75a-a0b4-4e72-a776-2776ece95606-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252124000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":1,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:03.793Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":1,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:03.793Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252124000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
+{"commitInfo":{"timestamp":1656252124212,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"26079ba4-cd65-4421-9bd3-ba0416dfb509"}}
diff --git a/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000003.json b/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000003.json
@@ -0,0 +1,2 @@
+{"add":{"path":"part-00000-b97608c1-8d8e-4369-b067-bd84435a1606-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252125000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":2,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:05.077Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":2,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:05.077Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252125000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
+{"commitInfo":{"timestamp":1656252125457,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"c2a690f3-3bb7-4dc2-b00e-cd81382e8b21"}}
diff --git a/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000004.json b/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000004.json
@@ -0,0 +1,2 @@
+{"add":{"path":"part-00000-7c59d077-8928-4402-b5e6-7259f5440fd0-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252126000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":3,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:06.242Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":3,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:06.242Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252126000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
+{"commitInfo":{"timestamp":1656252126676,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"4522c45a-ed48-4b7d-bfc6-1bad78981261"}}
diff --git a/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000005.json b/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000005.json
@@ -0,0 +1,2 @@
+{"add":{"path":"part-00000-7264b4fa-c3d7-4e25-956f-716358f594ff-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252127000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":4,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:07.511Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":4,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:07.511Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252127000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
+{"commitInfo":{"timestamp":1656252127961,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":4,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"f60b53dc-cb04-4e5b-a541-037c9c2cee89"}}
diff --git a/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000006.json b/rust/tests/data/delta-1.2.1-only-struct-stats/_delta_log/00000000000000000006.json
@@ -0,0 +1,2 @@
+{"add":{"path":"part-00000-1a04af5a-e0d5-497d-9496-7bcf9af3419f-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252129000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":5,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:08.788Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":5,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:08.788Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252129000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
+{"commitInfo":{"timestamp":1656252129258,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":5,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"0bf1e2ce-ef4f-4235-aa4c-918507d5097b"}}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"add":{"path":"part-00000-a222e75a-a0b4-4e72-a776-2776ece95606-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252124000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":1,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:03.793Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":1,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:03.793Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252124000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
		{"commitInfo":{"timestamp":1656252124212,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"26079ba4-cd65-4421-9bd3-ba0416dfb509"}}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"add":{"path":"part-00000-b97608c1-8d8e-4369-b067-bd84435a1606-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252125000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":2,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:05.077Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":2,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:05.077Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252125000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
		{"commitInfo":{"timestamp":1656252125457,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"c2a690f3-3bb7-4dc2-b00e-cd81382e8b21"}}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"add":{"path":"part-00000-7c59d077-8928-4402-b5e6-7259f5440fd0-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252126000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":3,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:06.242Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":3,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:06.242Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252126000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
		{"commitInfo":{"timestamp":1656252126676,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"4522c45a-ed48-4b7d-bfc6-1bad78981261"}}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"add":{"path":"part-00000-7264b4fa-c3d7-4e25-956f-716358f594ff-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252127000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":4,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:07.511Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":4,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:07.511Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252127000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
		{"commitInfo":{"timestamp":1656252127961,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":4,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"f60b53dc-cb04-4e5b-a541-037c9c2cee89"}}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"add":{"path":"part-00000-1a04af5a-e0d5-497d-9496-7bcf9af3419f-c000.snappy.parquet","partitionValues":{},"size":5489,"modificationTime":1656252129000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"integer\":5,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:08.788Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"maxValues\":{\"integer\":5,\"double\":1.234,\"decimal\":-5.67800,\"string\":\"string\",\"date\":\"2022-06-26\",\"timestamp\":\"2022-06-26T14:02:08.788Z\",\"struct\":{\"struct_element\":\"struct_value\"},\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":\"nested_struct_value\"}}},\"nullCount\":{\"integer\":0,\"null\":1,\"boolean\":0,\"double\":0,\"decimal\":0,\"string\":0,\"binary\":0,\"date\":0,\"timestamp\":0,\"struct\":{\"struct_element\":0},\"map\":0,\"array\":0,\"nested_struct\":{\"struct_element\":{\"nested_struct_element\":0}},\"struct_of_array_of_map\":{\"struct_element\":0}}}","tags":{"INSERTION_TIME":"1656252129000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}
		{"commitInfo":{"timestamp":1656252129258,"userId":"6114986638742036","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1829280694121074"},"clusterId":"0622-151429-s7rz8ws","readVersion":5,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5489"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"0bf1e2ce-ef4f-4235-aa4c-918507d5097b"}}