Merge branch 'main' into refactor/write_deltalake

README.md

@@ -1,6 +1,6 @@
 <p align="center">
   <a href="https://delta.io/">
-    <img src="https://github.com/delta-io/delta-rs/blob/main/logo.png?raw=true" alt="delta-rs logo" height="250">
+    <img src="https://github.com/delta-io/delta-rs/blob/main/docs\delta-rust-no-whitespace.svg?raw=true" alt="delta-rs logo" height="200">
   </a>
 </p>
 <p align="center">

crates/deltalake-aws/Cargo.toml → crates/aws/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-deltalake-core = { path = "../deltalake-core" }
+deltalake-core = { version = "0.17.0", path = "../core" }
 rusoto_core = { version = "0.47", default-features = false, optional = true }
 rusoto_credential = { version = "0.47" }
 rusoto_sts = { version = "0.47", default-features = false, optional = true }
@@ -26,10 +26,10 @@ url = { workspace = true }
 backoff = { version = "0.4", features = [ "tokio" ] }
 
 [dev-dependencies]
-deltalake-core = { path = "../deltalake-core", features = ["datafusion"] }
+deltalake-core = { path = "../core", features = ["datafusion"] }
 chrono = { workspace = true }
 serial_test = "3"
-deltalake-test = { path = "../deltalake-test" }
+deltalake-test = { path = "../test" }
 pretty_env_logger = "*"
 rand = "0.8"
 serde_json = { workspace = true }

crates/deltalake-azure/Cargo.toml → crates/azure/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-deltalake-core = { path = "../deltalake-core" }
+deltalake-core = { version = "0.17.0", path = "../core" }
 lazy_static = "1"
 
 # workspace depenndecies
@@ -21,7 +21,7 @@ url = { workspace = true }
 [dev-dependencies]
 chrono = { workspace = true }
 serial_test = "3"
-deltalake-test = { path = "../deltalake-test" }
+deltalake-test = { path = "../test" }
 pretty_env_logger = "*"
 rand = "0.8"
 serde_json = { workspace = true }

crates/benchmarks/Cargo.toml

@@ -41,6 +41,6 @@ datafusion-sql = { workspace = true }
 datafusion-physical-expr = { workspace = true }
 
 [dependencies.deltalake-core]
-path = "../deltalake-core"
+path = "../core"
 version = "0"
 features = ["datafusion"]

crates/deltalake-catalog-glue/Cargo.toml → crates/catalog-glue/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 async-trait = { workspace = true }
 aws-config = "1"
 aws-sdk-glue = "1"
-deltalake-core = { path = "../deltalake-core" }
+deltalake-core = { version = "0.17.0", path = "../core" }
 # This can depend on a lowest common denominator of core once that's released
 # deltalake_core = { version = "0.17.0" }
 thiserror = { workspace = true }

crates/deltalake-core/Cargo.toml → crates/core/Cargo.toml

@@ -102,7 +102,7 @@ sqlparser = { version = "0.40", optional = true }
 [dev-dependencies]
 criterion = "0.5"
 ctor = "0"
-deltalake-test = { path = "../deltalake-test", features = ["datafusion"] }
+deltalake-test = { path = "../test", features = ["datafusion"] }
 dotenvy = "0"
 hyper = { version = "0.14", features = ["server"] }
 maplit = "1"

crates/deltalake-core/src/data_catalog/storage/mod.rs → crates/core/src/data_catalog/storage/mod.rs

@@ -163,7 +163,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_table_names() {
-        let fs = ListingSchemaProvider::try_new("../deltalake-test/tests/data/", None).unwrap();
+        let fs = ListingSchemaProvider::try_new("../test/tests/data/", None).unwrap();
         fs.refresh().await.unwrap();
         let table_names = fs.table_names();
         assert!(table_names.len() > 20);
@@ -172,9 +172,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_query_table() {
-        let schema = Arc::new(
-            ListingSchemaProvider::try_new("../deltalake-test/tests/data/", None).unwrap(),
-        );
+        let schema = Arc::new(ListingSchemaProvider::try_new("../test/tests/data/", None).unwrap());
         schema.refresh().await.unwrap();
 
         let ctx = SessionContext::new();

crates/deltalake-core/src/delta_datafusion/mod.rs → crates/core/src/delta_datafusion/mod.rs

@@ -1689,7 +1689,7 @@ mod tests {
 
     #[tokio::test]
     async fn delta_table_provider_with_config() {
-        let table = crate::open_table("../deltalake-test/tests/data/delta-2.2.0-partitioned-types")
+        let table = crate::open_table("../test/tests/data/delta-2.2.0-partitioned-types")
             .await
             .unwrap();
         let config = DeltaScanConfigBuilder::new()

crates/deltalake-core/src/kernel/models/actions.rs → crates/core/src/kernel/models/actions.rs

@@ -955,10 +955,8 @@ mod tests {
         let inline = dv_inline();
         assert_eq!(None, inline.absolute_path(&parent).unwrap());
 
-        let path = std::fs::canonicalize(PathBuf::from(
-            "../deltalake-test/tests/data/table-with-dv-small/",
-        ))
-        .unwrap();
+        let path = std::fs::canonicalize(PathBuf::from("../test/tests/data/table-with-dv-small/"))
+            .unwrap();
         let parent = url::Url::from_directory_path(path).unwrap();
         let dv_url = parent
             .join("deletion_vector_61d16c75-6994-46b7-a15b-8b538852e50e.bin")

crates/deltalake-core/src/kernel/snapshot/log_data.rs → crates/core/src/kernel/snapshot/log_data.rs

@@ -682,7 +682,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_1_2_1_struct_stats_table() {
-        let table_uri = "../deltalake-test/tests/data/delta-1.2.1-only-struct-stats";
+        let table_uri = "../test/tests/data/delta-1.2.1-only-struct-stats";
         let table_from_struct_stats = crate::open_table(table_uri).await.unwrap();
         let table_from_json_stats = crate::open_table_with_version(table_uri, 1).await.unwrap();
 
@@ -727,7 +727,7 @@ mod tests {
 
     #[tokio::test]
     async fn df_stats_delta_1_2_1_struct_stats_table() {
-        let table_uri = "../deltalake-test/tests/data/delta-1.2.1-only-struct-stats";
+        let table_uri = "../test/tests/data/delta-1.2.1-only-struct-stats";
         let table_from_struct_stats = crate::open_table(table_uri).await.unwrap();
 
         let file_stats = table_from_struct_stats

crates/deltalake-core/src/lib.rs → crates/core/src/lib.rs

@@ -6,7 +6,7 @@
 //!
 //! ```rust
 //! async {
-//!   let table = deltalake_core::open_table("../deltalake-test/tests/data/simple_table").await.unwrap();
+//!   let table = deltalake_core::open_table("../test/tests/data/simple_table").await.unwrap();
 //!   let version = table.version();
 //! };
 //! ```
@@ -15,7 +15,7 @@
 //!
 //! ```rust
 //! async {
-//!   let table = deltalake_core::open_table_with_version("../deltalake-test/tests/data/simple_table", 0).await.unwrap();
+//!   let table = deltalake_core::open_table_with_version("../test/tests/data/simple_table", 0).await.unwrap();
 //!   let files = table.get_files_by_partitions(&[deltalake_core::PartitionFilter {
 //!       key: "month".to_string(),
 //!       value: deltalake_core::PartitionValue::Equal("12".to_string()),
@@ -28,7 +28,7 @@
 //! ```rust
 //! async {
 //!   let table = deltalake_core::open_table_with_ds(
-//!       "../deltalake-test/tests/data/simple_table",
+//!       "../test/tests/data/simple_table",
 //!       "2020-05-02T23:47:31-07:00",
 //!   ).await.unwrap();
 //!   let version = table.version();
@@ -53,7 +53,7 @@
 //!
 //! async {
 //!   let mut ctx = SessionContext::new();
-//!   let table = deltalake_core::open_table("../deltalake-test/tests/data/simple_table")
+//!   let table = deltalake_core::open_table("../test/tests/data/simple_table")
 //!       .await
 //!       .unwrap();
 //!   ctx.register_table("demo", Arc::new(table)).unwrap();
@@ -174,7 +174,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_2_0_table_without_version() {
-        let table = crate::open_table("../deltalake-test/tests/data/delta-0.2.0")
+        let table = crate::open_table("../test/tests/data/delta-0.2.0")
             .await
             .unwrap();
         assert_eq!(table.version(), 3);
@@ -212,7 +212,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_table_with_update() {
-        let path = "../deltalake-test/tests/data/simple_table_with_checkpoint/";
+        let path = "../test/tests/data/simple_table_with_checkpoint/";
         let table_newest_version = crate::open_table(path).await.unwrap();
         let mut table_to_update = crate::open_table_with_version(path, 0).await.unwrap();
         // calling update several times should not produce any duplicates
@@ -227,10 +227,9 @@ mod tests {
     }
     #[tokio::test]
     async fn read_delta_2_0_table_with_version() {
-        let mut table =
-            crate::open_table_with_version("../deltalake-test/tests/data/delta-0.2.0", 0)
-                .await
-                .unwrap();
+        let mut table = crate::open_table_with_version("../test/tests/data/delta-0.2.0", 0)
+            .await
+            .unwrap();
         assert_eq!(table.version(), 0);
         assert_eq!(table.protocol().unwrap().min_writer_version, 2);
         assert_eq!(table.protocol().unwrap().min_reader_version, 1);
@@ -242,7 +241,7 @@ mod tests {
             ],
         );
 
-        table = crate::open_table_with_version("../deltalake-test/tests/data/delta-0.2.0", 2)
+        table = crate::open_table_with_version("../test/tests/data/delta-0.2.0", 2)
             .await
             .unwrap();
         assert_eq!(table.version(), 2);
@@ -256,7 +255,7 @@ mod tests {
             ]
         );
 
-        table = crate::open_table_with_version("../deltalake-test/tests/data/delta-0.2.0", 3)
+        table = crate::open_table_with_version("../test/tests/data/delta-0.2.0", 3)
             .await
             .unwrap();
         assert_eq!(table.version(), 3);
@@ -274,7 +273,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_8_0_table_without_version() {
-        let table = crate::open_table("../deltalake-test/tests/data/delta-0.8.0")
+        let table = crate::open_table("../test/tests/data/delta-0.8.0")
             .await
             .unwrap();
         assert_eq!(table.version(), 1);
@@ -330,7 +329,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_8_0_table_with_load_version() {
-        let mut table = crate::open_table("../deltalake-test/tests/data/delta-0.8.0")
+        let mut table = crate::open_table("../test/tests/data/delta-0.8.0")
             .await
             .unwrap();
         assert_eq!(table.version(), 1);
@@ -358,7 +357,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_8_0_table_with_partitions() {
-        let table = crate::open_table("../deltalake-test/tests/data/delta-0.8.0-partitioned")
+        let table = crate::open_table("../test/tests/data/delta-0.8.0-partitioned")
             .await
             .unwrap();
 
@@ -383,8 +382,8 @@ mod tests {
         assert_eq!(
             table.get_file_uris_by_partitions(&filters).unwrap().into_iter().map(|p| std::fs::canonicalize(p).unwrap()).collect::<Vec<_>>(),
             vec![
-                std::fs::canonicalize("../deltalake-test/tests/data/delta-0.8.0-partitioned/year=2020/month=2/day=3/part-00000-94d16827-f2fd-42cd-a060-f67ccc63ced9.c000.snappy.parquet").unwrap(),
-                std::fs::canonicalize("../deltalake-test/tests/data/delta-0.8.0-partitioned/year=2020/month=2/day=5/part-00000-89cdd4c8-2af7-4add-8ea3-3990b2f027b5.c000.snappy.parquet").unwrap(),
+                std::fs::canonicalize("../test/tests/data/delta-0.8.0-partitioned/year=2020/month=2/day=3/part-00000-94d16827-f2fd-42cd-a060-f67ccc63ced9.c000.snappy.parquet").unwrap(),
+                std::fs::canonicalize("../test/tests/data/delta-0.8.0-partitioned/year=2020/month=2/day=5/part-00000-89cdd4c8-2af7-4add-8ea3-3990b2f027b5.c000.snappy.parquet").unwrap(),
             ]
         );
 
@@ -431,7 +430,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_8_0_table_with_null_partition() {
-        let table = crate::open_table("../deltalake-test/tests/data/delta-0.8.0-null-partition")
+        let table = crate::open_table("../test/tests/data/delta-0.8.0-null-partition")
             .await
             .unwrap();
 
@@ -460,7 +459,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_8_0_table_with_special_partition() {
-        let table = crate::open_table("../deltalake-test/tests/data/delta-0.8.0-special-partition")
+        let table = crate::open_table("../test/tests/data/delta-0.8.0-special-partition")
             .await
             .unwrap();
 
@@ -493,7 +492,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_8_0_table_partition_with_compare_op() {
-        let table = crate::open_table("../deltalake-test/tests/data/delta-0.8.0-numeric-partition")
+        let table = crate::open_table("../test/tests/data/delta-0.8.0-numeric-partition")
             .await
             .unwrap();
 
@@ -522,7 +521,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_table_history() {
-        let path = "../deltalake-test/tests/data/simple_table_with_checkpoint";
+        let path = "../test/tests/data/simple_table_with_checkpoint";
         let latest_table = crate::open_table(path).await.unwrap();
 
         let table = crate::open_table_with_version(path, 1).await.unwrap();
@@ -544,7 +543,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_poll_table_commits() {
-        let path = "../deltalake-test/tests/data/simple_table_with_checkpoint";
+        let path = "../test/tests/data/simple_table_with_checkpoint";
         let mut table = crate::open_table_with_version(path, 9).await.unwrap();
         let peek = table.peek_next_commit(table.version()).await.unwrap();
         assert!(matches!(peek, PeekCommit::New(..)));
@@ -574,14 +573,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_read_vacuumed_log() {
-        let path = "../deltalake-test/tests/data/checkpoints_vacuumed";
+        let path = "../test/tests/data/checkpoints_vacuumed";
         let table = crate::open_table(path).await.unwrap();
         assert_eq!(table.version(), 12);
     }
 
     #[tokio::test]
     async fn test_read_vacuumed_log_history() {
-        let path = "../deltalake-test/tests/data/checkpoints_vacuumed";
+        let path = "../test/tests/data/checkpoints_vacuumed";
         let table = crate::open_table(path).await.unwrap();
 
         // load history for table version with available log file
@@ -626,7 +625,7 @@ mod tests {
 
     #[tokio::test]
     async fn read_delta_table_with_cdc() {
-        let table = crate::open_table("../deltalake-test/tests/data/simple_table_with_cdc")
+        let table = crate::open_table("../test/tests/data/simple_table_with_cdc")
             .await
             .unwrap();
         assert_eq!(table.version(), 2);
@@ -640,7 +639,7 @@ mod tests {
 
     #[tokio::test()]
     async fn test_version_zero_table_load() {
-        let path = "../deltalake-test/tests/data/COVID-19_NYT";
+        let path = "../test/tests/data/COVID-19_NYT";
         let latest_table: DeltaTable = crate::open_table(path).await.unwrap();
 
         let version_0_table = crate::open_table_with_version(path, 0).await.unwrap();
@@ -661,7 +660,7 @@ mod tests {
     async fn test_fail_fast_on_not_existing_path() {
         use std::path::Path as FolderPath;
 
-        let non_existing_path_str = "../deltalake-test/tests/data/folder_doesnt_exist";
+        let non_existing_path_str = "../test/tests/data/folder_doesnt_exist";
 
         // Check that there is no such path at the beginning
         let path_doesnt_exist = !FolderPath::new(non_existing_path_str).exists();