From 80f71500314ef414e77dd61c3c06c0448bfa5166 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Thu, 25 Apr 2024 08:40:11 +0200 Subject: [PATCH 1/3] reuse table state in rust engine py-writer --- python/deltalake/_internal.pyi | 1 + python/deltalake/writer.py | 1 + python/src/lib.rs | 9 +++++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/python/deltalake/_internal.pyi b/python/deltalake/_internal.pyi index 7d2e6342a2..551528961e 100644 --- a/python/deltalake/_internal.pyi +++ b/python/deltalake/_internal.pyi @@ -174,6 +174,7 @@ def write_to_deltalake( data: pyarrow.RecordBatchReader, partition_by: Optional[List[str]], mode: str, + table: Optional[RawDeltaTable], schema_mode: Optional[str], predicate: Optional[str], name: Optional[str], diff --git a/python/deltalake/writer.py b/python/deltalake/writer.py index 2bc299215b..e4dd3fd42d 100644 --- a/python/deltalake/writer.py +++ b/python/deltalake/writer.py @@ -318,6 +318,7 @@ def write_deltalake( data=data, partition_by=partition_by, mode=mode, + table=table._table if table is not None else None, schema_mode=schema_mode, predicate=predicate, name=name, diff --git a/python/src/lib.rs b/python/src/lib.rs index 917bbb5750..abb1a5619d 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -1410,6 +1410,7 @@ fn write_to_deltalake( table_uri: String, data: PyArrowType, mode: String, + table: Option<&RawDeltaTable>, schema_mode: Option, partition_by: Option>, predicate: Option, @@ -1425,11 +1426,15 @@ fn write_to_deltalake( let save_mode = mode.parse().map_err(PythonError::from)?; let options = storage_options.clone().unwrap_or_default(); - let table = rt() + let table = if let Some(table) = table { + DeltaOps(table._table.clone()) + } else { + rt() .block_on(DeltaOps::try_from_uri_with_storage_options( &table_uri, options, )) - .map_err(PythonError::from)?; + .map_err(PythonError::from)? + }; let mut builder = table.write(batches).with_save_mode(save_mode); if let Some(schema_mode) = schema_mode { From 255806a8ee80b37384d86f29b6d3888cfc7599e8 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Thu, 25 Apr 2024 08:42:15 +0200 Subject: [PATCH 2/3] fmt --- python/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/src/lib.rs b/python/src/lib.rs index abb1a5619d..5d3e813e33 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -1429,8 +1429,7 @@ fn write_to_deltalake( let table = if let Some(table) = table { DeltaOps(table._table.clone()) } else { - rt() - .block_on(DeltaOps::try_from_uri_with_storage_options( + rt().block_on(DeltaOps::try_from_uri_with_storage_options( &table_uri, options, )) .map_err(PythonError::from)? From 60e28259d41046492cc2ccc7c092102753c99d12 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:34:14 +0200 Subject: [PATCH 3/3] quick fix release --- python/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Cargo.toml b/python/Cargo.toml index 1589ecb11e..0c205dc20f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake-python" -version = "0.17.1" +version = "0.17.2" authors = ["Qingping Hou ", "Will Jones "] homepage = "https://github.com/delta-io/delta-rs" license = "Apache-2.0"