Update arrow 47.0.0

apache · Sep 18, 2023 · c00628d · c00628d
1 parent c72b98e
commit c00628d
Show file tree

Hide file tree

Showing 18 changed files with 136 additions and 273 deletions.
diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml
@@ -47,7 +47,6 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      # Note: this does not include dictionary_expressions to reduce codegen
       - name: Run doctests
         run: cargo test --doc --features avro,json
       - name: Verify Working Directory Clean

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -65,7 +65,6 @@ jobs:
       - name: Check workspace in debug mode
         run: cargo check
 
-      # Note: this does not include dictionary_expressions to reduce codegen
       - name: Check workspace with all features
         run: cargo check --workspace --benches --features avro,json
       - name: Check Cargo.lock for datafusion-cli
@@ -96,7 +95,7 @@ jobs:
         with:
           rust-version: stable
       - name: Run tests (excluding doctests)
-        run: cargo test --lib --tests --bins --features avro,json,dictionary_expressions,backtrace
+        run: cargo test --lib --tests --bins --features avro,json,backtrace
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -177,7 +176,6 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      # Note: this does not include dictionary_expressions to reduce codegen
       - name: Run doctests
         run: |
           cargo test --doc --features avro,json
@@ -302,7 +300,7 @@ jobs:
         shell: bash
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
-          cargo test --lib --tests --bins --features avro,json,dictionary_expressions,backtrace
+          cargo test --lib --tests --bins --features avro,json,backtrace
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
         env:
@@ -338,7 +336,7 @@ jobs:
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
-          cargo test --lib --tests --bins --features avro,json,dictionary_expressions,backtrace
+          cargo test --lib --tests --bins --features avro,json,backtrace
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
         env:

diff --git a/Cargo.toml b/Cargo.toml
@@ -32,7 +32,7 @@ rust-version = "1.70"
 version = "31.0.0"
 
 [workspace.dependencies]
-arrow = { version = "46.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
+arrow = { version = "46.0.0", features = ["prettyprint"] }
 arrow-array = { version = "46.0.0", default-features = false, features = ["chrono-tz"] }
 arrow-buffer = { version = "46.0.0", default-features = false }
 arrow-flight = { version = "46.0.0", features = ["flight-sql-experimental"] }
@@ -41,7 +41,6 @@ parquet = { version = "46.0.0", features = ["arrow", "async", "object_store"] }
 sqlparser = { version = "0.37.0", features = ["visitor"] }
 chrono = { version = "0.4.31", default-features = false }
 
-
 [profile.release]
 codegen-units = 1
 lto = true
@@ -59,3 +58,11 @@ opt-level = 3
 overflow-checks = false
 panic = 'unwind'
 rpath = false
+
+[patch.crates-io]
+arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -50,3 +50,10 @@ assert_cmd = "2.0"
 ctor = "0.2.0"
 predicates = "3.0"
 rstest = "0.17"
+
+[patch.crates-io]
+arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
+parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "175c7765939c0738defc736426c0b0a93b00bfa8" }
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
@@ -47,12 +47,12 @@ log = "0.4"
 mimalloc = { version = "0.1", default-features = false }
 num_cpus = "1.13.0"
 object_store = { version = "0.7.0", features = ["aws"] }
-prost = { version = "0.11", default-features = false }
+prost = { version = "0.12", default-features = false }
 prost-derive = { version = "0.11", default-features = false }
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.82"
 tempfile = "3"
 tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
-tonic = "0.9"
+tonic = "0.10"
 url = "2.2"
 uuid = "1.2"
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
@@ -40,9 +40,6 @@ backtrace = ["datafusion-common/backtrace"]
 compression = ["xz2", "bzip2", "flate2", "zstd", "async-compression"]
 crypto_expressions = ["datafusion-physical-expr/crypto_expressions", "datafusion-optimizer/crypto_expressions"]
 default = ["crypto_expressions", "encoding__expressions", "regex_expressions", "unicode_expressions", "compression"]
-# Enables support for non-scalar, binary operations on dictionaries
-# Note: this results in significant additional codegen
-dictionary_expressions = ["datafusion-physical-expr/dictionary_expressions", "datafusion-optimizer/dictionary_expressions"]
 encoding__expressions = ["datafusion-physical-expr/encoding_expressions"]
 # Used for testing ONLY: causes all values to hash to the same value (test for collisions)
 force_hash_collisions = []

diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs
@@ -188,7 +188,6 @@ async fn query_nested_get_indexed_field_on_struct() -> Result<()> {
 }
 
 #[tokio::test]
-#[cfg(feature = "dictionary_expressions")]
 async fn query_on_string_dictionary() -> Result<()> {
     // Test to ensure DataFusion can operate on dictionary types
     // Use StringDictionary (32 bit indexes = keys)

diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
@@ -35,7 +35,6 @@ path = "src/lib.rs"
 [features]
 crypto_expressions = ["datafusion-physical-expr/crypto_expressions"]
 default = ["unicode_expressions", "crypto_expressions", "regex_expressions"]
-dictionary_expressions = ["datafusion-physical-expr/dictionary_expressions"]
 regex_expressions = ["datafusion-physical-expr/regex_expressions"]
 unicode_expressions = ["datafusion-physical-expr/unicode_expressions"]
 

diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
@@ -35,9 +35,6 @@ path = "src/lib.rs"
 [features]
 crypto_expressions = ["md-5", "sha2", "blake2", "blake3"]
 default = ["crypto_expressions", "regex_expressions", "unicode_expressions", "encoding_expressions"]
-# Enables support for non-scalar, binary operations on dictionaries
-# Note: this results in significant additional codegen
-dictionary_expressions = ["arrow/dyn_cmp_dict"]
 encoding_expressions = ["base64", "hex"]
 regex_expressions = ["regex"]
 unicode_expressions = ["unicode-segmentation"]