From 6532b97f99581295a5253656d4ea4ea73849bc56 Mon Sep 17 00:00:00 2001 From: Peter Toth Date: Wed, 3 Jul 2024 17:38:28 +0200 Subject: [PATCH 1/6] Improve volatile expression handling in `CommonSubexprEliminate` rule --- datafusion-cli/Cargo.lock | 103 +++++++++--------- datafusion/expr/src/expr.rs | 10 +- datafusion/optimizer/Cargo.toml | 1 + .../optimizer/src/common_subexpr_eliminate.rs | 94 ++++++++++------ 4 files changed, 121 insertions(+), 87 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 5fc8dbcfdfb3..380723bd72a5 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -875,9 +875,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2755ff20a1d93490d26ba33a6f092a38a508398a5320df5d4b3014fcccce9410" +checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" dependencies = [ "jobserver", "libc", @@ -900,7 +900,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -1333,6 +1333,7 @@ dependencies = [ "chrono", "datafusion-common", "datafusion-expr", + "datafusion-functions", "datafusion-physical-expr", "hashbrown 0.14.5", "indexmap 2.2.6", @@ -1964,9 +1965,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" dependencies = [ "bytes", "futures-channel", @@ -2005,10 +2006,10 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-util", "rustls 0.23.10", - "rustls-native-certs 0.7.0", + "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -2017,16 +2018,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.3.1", + "hyper 1.4.0", "pin-project-lite", "socket2", "tokio", @@ -2501,7 +2502,7 @@ dependencies = [ "chrono", "futures", "humantime", - "hyper 1.3.1", + "hyper 1.4.0", "itertools", "md-5", "parking_lot", @@ -2573,7 +2574,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2975,7 +2976,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-rustls 0.27.2", "hyper-util", "ipnet", @@ -2987,7 +2988,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.10", - "rustls-native-certs 0.7.0", + "rustls-native-certs 0.7.1", "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", @@ -3142,9 +3143,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile 2.1.2", @@ -3180,9 +3181,9 @@ checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -3315,9 +3316,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.119" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8eddb61f0697cc3989c5d64b452f5488e2b8a60fd7d5076a3045076ffef8cb0" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", @@ -4097,7 +4098,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4115,7 +4116,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4135,18 +4136,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -4157,9 +4158,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -4169,9 +4170,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -4181,15 +4182,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -4199,9 +4200,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -4211,9 +4212,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -4223,9 +4224,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -4235,9 +4236,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winreg" @@ -4266,18 +4267,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index 579f5fed578f..fd0d1e41612b 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -1413,12 +1413,16 @@ impl Expr { .unwrap() } + /// Returns true if the expression node is volatile, i.e. whether it can return + /// different results when evaluated multiple times with the same input. + pub fn is_volatile_node(&self) -> bool { + matches!(self, Expr::ScalarFunction(func) if func.func.signature().volatility == Volatility::Volatile) + } + /// Returns true if the expression is volatile, i.e. whether it can return different /// results when evaluated multiple times with the same input. pub fn is_volatile(&self) -> Result { - self.exists(|expr| { - Ok(matches!(expr, Expr::ScalarFunction(func) if func.func.signature().volatility == Volatility::Volatile )) - }) + self.exists(|expr| Ok(expr.is_volatile_node())) } /// Recursively find all [`Expr::Placeholder`] expressions, and diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index 1a9e9630c076..54d8c472f13f 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -45,6 +45,7 @@ async-trait = { workspace = true } chrono = { workspace = true } datafusion-common = { workspace = true, default-features = true } datafusion-expr = { workspace = true } +datafusion-functions = { workspace = true } datafusion-physical-expr = { workspace = true } hashbrown = { workspace = true } indexmap = { workspace = true } diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs index cebae410f309..2bdcfc5cbe2a 100644 --- a/datafusion/optimizer/src/common_subexpr_eliminate.rs +++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs @@ -191,24 +191,19 @@ impl CommonSubexprEliminate { id_array: &mut IdArray<'n>, expr_mask: ExprMask, ) -> Result { - // Don't consider volatile expressions for CSE. - Ok(if expr.is_volatile()? { - false - } else { - let mut visitor = ExprIdentifierVisitor { - expr_stats, - id_array, - visit_stack: vec![], - down_index: 0, - up_index: 0, - expr_mask, - random_state: &self.random_state, - found_common: false, - }; - expr.visit(&mut visitor)?; + let mut visitor = ExprIdentifierVisitor { + expr_stats, + id_array, + visit_stack: vec![], + down_index: 0, + up_index: 0, + expr_mask, + random_state: &self.random_state, + found_common: false, + }; + expr.visit(&mut visitor)?; - visitor.found_common - }) + Ok(visitor.found_common) } /// Rewrites `exprs_list` with common sub-expressions replaced with a new @@ -917,27 +912,36 @@ struct ExprIdentifierVisitor<'a, 'n> { /// Record item that used when traversing an expression tree. enum VisitRecord<'n> { - /// Contains the post-order index assigned in during the first, visiting traversal and - /// a boolean flag to indicate if the record marks an expression subtree (not just a - /// single node). + /// Marks the beginning of expression. It contains: + /// - The post-order index assigned during the first, visiting traversal. + /// - A boolean flag if the record marks an expression subtree (not just a single + /// node). EnterMark(usize, bool), - /// Accumulated identifier of sub expression. - ExprItem(Identifier<'n>), + + /// Marks an accumulated subexpression tree. It contains: + /// - The accumulated identifier of a subexpression. + /// - A boolean flag if the expression is valid for subexpression elimination. + /// The flag is propagated up from children to parent. (E.g. volatile expressions + /// are not valid and can't be extracted, but non-volatile children of volatile + /// expressions can be extracted.) + ExprItem(Identifier<'n>, bool), } impl<'n> ExprIdentifierVisitor<'_, 'n> { /// Find the first `EnterMark` in the stack, and accumulates every `ExprItem` /// before it. - fn pop_enter_mark(&mut self) -> (usize, bool, Option>) { + fn pop_enter_mark(&mut self) -> (usize, bool, Option>, bool) { let mut expr_id = None; + let mut is_valid = true; while let Some(item) = self.visit_stack.pop() { match item { - VisitRecord::EnterMark(down_index, tree) => { - return (down_index, tree, expr_id); + VisitRecord::EnterMark(down_index, is_tree) => { + return (down_index, is_tree, expr_id, is_valid); } - VisitRecord::ExprItem(id) => { - expr_id = Some(id.combine(expr_id)); + VisitRecord::ExprItem(sub_expr_id, sub_expr_is_valid) => { + expr_id = Some(sub_expr_id.combine(expr_id)); + is_valid &= sub_expr_is_valid; } } } @@ -949,8 +953,6 @@ impl<'n> TreeNodeVisitor<'n> for ExprIdentifierVisitor<'_, 'n> { type Node = Expr; fn f_down(&mut self, expr: &'n Expr) -> Result { - // TODO: consider non-volatile sub-expressions for CSE - // If an expression can short circuit its children then don't consider its // children for CSE (https://github.com/apache/arrow-datafusion/issues/8814). // This means that we don't recurse into its children, but handle the expression @@ -972,13 +974,14 @@ impl<'n> TreeNodeVisitor<'n> for ExprIdentifierVisitor<'_, 'n> { } fn f_up(&mut self, expr: &'n Expr) -> Result { - let (down_index, is_tree, sub_expr_id) = self.pop_enter_mark(); + let (down_index, is_tree, sub_expr_id, sub_expr_is_valid) = self.pop_enter_mark(); let expr_id = Identifier::new(expr, is_tree, self.random_state).combine(sub_expr_id); + let is_valid = !expr.is_volatile_node() && sub_expr_is_valid; self.id_array[down_index].0 = self.up_index; - if !self.expr_mask.ignores(expr) { + if is_valid && !self.expr_mask.ignores(expr) { self.id_array[down_index].1 = Some(expr_id); let count = self.expr_stats.entry(expr_id).or_insert(0); *count += 1; @@ -986,7 +989,8 @@ impl<'n> TreeNodeVisitor<'n> for ExprIdentifierVisitor<'_, 'n> { self.found_common = true; } } - self.visit_stack.push(VisitRecord::ExprItem(expr_id)); + self.visit_stack + .push(VisitRecord::ExprItem(expr_id, is_valid)); self.up_index += 1; Ok(TreeNodeRecursion::Continue) @@ -1105,13 +1109,14 @@ mod test { use std::iter; use arrow::datatypes::{DataType, Field, Schema}; - use datafusion_expr::expr::AggregateFunction; + use datafusion_expr::expr::{AggregateFunction, ScalarFunction}; use datafusion_expr::logical_plan::{table_scan, JoinType}; use datafusion_expr::{ grouping_set, AccumulatorFactoryFunction, AggregateUDF, BinaryExpr, Signature, SimpleAggregateUDF, Volatility, }; use datafusion_expr::{lit, logical_plan::builder::LogicalPlanBuilder}; + use datafusion_functions::math; use crate::optimizer::OptimizerContext; use crate::test::*; @@ -1838,4 +1843,27 @@ mod test { Ok(()) } + + #[test] + fn test_volatile() -> Result<()> { + let table_scan = test_table_scan()?; + + let extracted_child = col("a") + col("b"); + let rand = Expr::ScalarFunction(ScalarFunction::new_udf(math::random(), vec![])); + let not_extracted_volatile = extracted_child + rand; + let plan = LogicalPlanBuilder::from(table_scan.clone()) + .project(vec![ + not_extracted_volatile.clone().alias("c1"), + not_extracted_volatile.alias("c2"), + ])? + .build()?; + + let expected = "Projection: __common_expr_1 + random() AS c1, __common_expr_1 + random() AS c2\ + \n Projection: test.a + test.b AS __common_expr_1, test.a, test.b, test.c\ + \n TableScan: test"; + + assert_optimized_plan_eq(expected, plan, None); + + Ok(()) + } } From 3c3802cf1cc7329dce8403153c45dddb8822c67f Mon Sep 17 00:00:00 2001 From: Peter Toth Date: Fri, 5 Jul 2024 18:01:35 +0200 Subject: [PATCH 2/6] fix volatile handling with short circuits --- .../optimizer/src/common_subexpr_eliminate.rs | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs index 2bdcfc5cbe2a..8a01a5139b8f 100644 --- a/datafusion/optimizer/src/common_subexpr_eliminate.rs +++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs @@ -976,9 +976,17 @@ impl<'n> TreeNodeVisitor<'n> for ExprIdentifierVisitor<'_, 'n> { fn f_up(&mut self, expr: &'n Expr) -> Result { let (down_index, is_tree, sub_expr_id, sub_expr_is_valid) = self.pop_enter_mark(); - let expr_id = - Identifier::new(expr, is_tree, self.random_state).combine(sub_expr_id); - let is_valid = !expr.is_volatile_node() && sub_expr_is_valid; + let (expr_id, is_valid) = if is_tree { + ( + Identifier::new(expr, true, self.random_state), + !expr.is_volatile()?, + ) + } else { + ( + Identifier::new(expr, false, self.random_state).combine(sub_expr_id), + !expr.is_volatile_node() && sub_expr_is_valid, + ) + }; self.id_array[down_index].0 = self.up_index; if is_valid && !self.expr_mask.ignores(expr) { @@ -1866,4 +1874,30 @@ mod test { Ok(()) } + + #[test] + fn test_volatile_short_circuits() -> Result<()> { + let table_scan = test_table_scan()?; + + let rand = Expr::ScalarFunction(ScalarFunction::new_udf(math::random(), vec![])); + let not_extracted_volatile_short_circuit_2 = + rand.clone().eq(lit(0)).or(col("b").eq(lit(0))); + let not_extracted_volatile_short_circuit_1 = + col("a").eq(lit(0)).or(rand.eq(lit(0))); + let plan = LogicalPlanBuilder::from(table_scan.clone()) + .project(vec![ + not_extracted_volatile_short_circuit_1.clone().alias("c1"), + not_extracted_volatile_short_circuit_1.alias("c2"), + not_extracted_volatile_short_circuit_2.clone().alias("c3"), + not_extracted_volatile_short_circuit_2.alias("c4"), + ])? + .build()?; + + let expected = "Projection: test.a = Int32(0) OR random() = Int32(0) AS c1, test.a = Int32(0) OR random() = Int32(0) AS c2, random() = Int32(0) OR test.b = Int32(0) AS c3, random() = Int32(0) OR test.b = Int32(0) AS c4\ + \n TableScan: test"; + + assert_non_optimized_plan_eq(expected, plan, None); + + Ok(()) + } } From 79b2e022678177358d8325996354ea10a34582fb Mon Sep 17 00:00:00 2001 From: Peter Toth Date: Fri, 5 Jul 2024 19:40:20 +0200 Subject: [PATCH 3/6] fix comments --- datafusion/expr/src/expr.rs | 3 +++ .../optimizer/src/common_subexpr_eliminate.rs | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index fd0d1e41612b..ecece6dbfce7 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -1415,6 +1415,9 @@ impl Expr { /// Returns true if the expression node is volatile, i.e. whether it can return /// different results when evaluated multiple times with the same input. + /// Note: unlike [`Self::is_volatile`], this function does not consider inputs: + /// - `rand()` returns `true`, + /// - `a + rand()` returns `false` pub fn is_volatile_node(&self) -> bool { matches!(self, Expr::ScalarFunction(func) if func.func.signature().volatility == Volatility::Volatile) } diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs index 8a01a5139b8f..d03cc361b9bc 100644 --- a/datafusion/optimizer/src/common_subexpr_eliminate.rs +++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs @@ -928,8 +928,22 @@ enum VisitRecord<'n> { } impl<'n> ExprIdentifierVisitor<'_, 'n> { - /// Find the first `EnterMark` in the stack, and accumulates every `ExprItem` - /// before it. + /// Find the first `EnterMark` in the stack, and accumulates every `ExprItem` before + /// it. Returns a tuple that contains: + /// - The pre-order index of the expression we marked. + /// - A boolean flag if we marked an expression subtree (not just a single node). + /// If true we didn't recurse into the node's children, so we need to calculate the + /// hash of the marked expression tree (not just the node) and we need to validate + /// the expression tree (not just the node). + /// - The accumulated identifier of the children of the marked expression. + /// - An accumulated boolean flag from the children of the marked expression if all + /// children are valid for subexpression elimination (i.e. it is safe to extract the + /// expression as a common expression from its children POV). + /// (E.g. if any of the children of the marked expression is not valid (e.g. is + /// volatile) then the expression is also not valid, so we can propagate this + /// information up from children to parents via `visit_stack` during the first, + /// visiting traversal and no need to test the expression's validity beforehand with + /// an extra traversal). fn pop_enter_mark(&mut self) -> (usize, bool, Option>, bool) { let mut expr_id = None; let mut is_valid = true; From 1aa88486775266570c7219d490201eb96a8dfc4d Mon Sep 17 00:00:00 2001 From: Peter Toth Date: Sat, 6 Jul 2024 10:55:27 +0200 Subject: [PATCH 4/6] add slt tests for CSE --- datafusion/sqllogictest/test_files/cse.slt | 173 +++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 datafusion/sqllogictest/test_files/cse.slt diff --git a/datafusion/sqllogictest/test_files/cse.slt b/datafusion/sqllogictest/test_files/cse.slt new file mode 100644 index 000000000000..3579c1c1635c --- /dev/null +++ b/datafusion/sqllogictest/test_files/cse.slt @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +statement ok +CREATE TABLE IF NOT EXISTS t1(a DOUBLE, b DOUBLE) + +# Trivial common expression +query TT +EXPLAIN SELECT + a + 1 AS c1, + a + 1 AS c2 +FROM t1 +---- +logical_plan +01)Projection: __common_expr_1 AS c1, __common_expr_1 AS c2 +02)--Projection: t1.a + Float64(1) AS __common_expr_1 +03)----TableScan: t1 projection=[a] +physical_plan +01)ProjectionExec: expr=[__common_expr_1@0 as c1, __common_expr_1@0 as c2] +02)--ProjectionExec: expr=[a@0 + 1 as __common_expr_1] +03)----MemoryExec: partitions=1, partition_sizes=[0] + +# Common volatile expression +query TT +EXPLAIN SELECT + a + random() AS c1, + a + random() AS c2 +FROM t1 +---- +logical_plan +01)Projection: t1.a + random() AS c1, t1.a + random() AS c2 +02)--TableScan: t1 projection=[a] +physical_plan +01)ProjectionExec: expr=[a@0 + random() as c1, a@0 + random() as c2] +02)--MemoryExec: partitions=1, partition_sizes=[0] + +# Volatile expression with non-volatile common child +query TT +EXPLAIN SELECT + a + 1 + random() AS c1, + a + 1 + random() AS c2 +FROM t1 +---- +logical_plan +01)Projection: __common_expr_1 + random() AS c1, __common_expr_1 + random() AS c2 +02)--Projection: t1.a + Float64(1) AS __common_expr_1 +03)----TableScan: t1 projection=[a] +physical_plan +01)ProjectionExec: expr=[__common_expr_1@0 + random() as c1, __common_expr_1@0 + random() as c2] +02)--ProjectionExec: expr=[a@0 + 1 as __common_expr_1] +03)----MemoryExec: partitions=1, partition_sizes=[0] + +# Volatile expression with non-volatile common children +query TT +EXPLAIN SELECT + a + 1 + random() + (a + 2) AS c1, + a + 1 + random() + (a + 2) AS c2 +FROM t1 +---- +logical_plan +01)Projection: __common_expr_1 + random() + __common_expr_2 AS c1, __common_expr_1 + random() + __common_expr_2 AS c2 +02)--Projection: t1.a + Float64(1) AS __common_expr_1, t1.a + Float64(2) AS __common_expr_2 +03)----TableScan: t1 projection=[a] +physical_plan +01)ProjectionExec: expr=[__common_expr_1@0 + random() + __common_expr_2@1 as c1, __common_expr_1@0 + random() + __common_expr_2@1 as c2] +02)--ProjectionExec: expr=[a@0 + 1 as __common_expr_1, a@0 + 2 as __common_expr_2] +03)----MemoryExec: partitions=1, partition_sizes=[0] + +# Common short-circuit expression +query TT +EXPLAIN SELECT + a = 0 AND b = 0 AS c1, + a = 0 AND b = 0 AS c2, + a = 0 OR b = 0 AS c3, + a = 0 OR b = 0 AS c4, + CASE WHEN (a = 0) THEN 0 ELSE 1 END AS c5, + CASE WHEN (a = 0) THEN 0 ELSE 1 END AS c6 +FROM t1 +---- +logical_plan +01)Projection: __common_expr_1 AS c1, __common_expr_1 AS c2, __common_expr_2 AS c3, __common_expr_2 AS c4, __common_expr_3 AS c5, __common_expr_3 AS c6 +02)--Projection: t1.a = Float64(0) AND t1.b = Float64(0) AS __common_expr_1, t1.a = Float64(0) OR t1.b = Float64(0) AS __common_expr_2, CASE WHEN t1.a = Float64(0) THEN Int64(0) ELSE Int64(1) END AS __common_expr_3 +03)----TableScan: t1 projection=[a, b] +physical_plan +01)ProjectionExec: expr=[__common_expr_1@0 as c1, __common_expr_1@0 as c2, __common_expr_2@1 as c3, __common_expr_2@1 as c4, __common_expr_3@2 as c5, __common_expr_3@2 as c6] +02)--ProjectionExec: expr=[a@0 = 0 AND b@1 = 0 as __common_expr_1, a@0 = 0 OR b@1 = 0 as __common_expr_2, CASE WHEN a@0 = 0 THEN 0 ELSE 1 END as __common_expr_3] +03)----MemoryExec: partitions=1, partition_sizes=[0] + +# Common children of short-circuit expression +# TODO: consider surely executed children of "short circuited"s for CSE. i.e. `a = 0`, `a = 2`, `a = 4` should be extracted +query TT +EXPLAIN SELECT + a = 0 AND b = 0 AS c1, + a = 0 AND b = 1 AS c2, + b = 2 AND a = 1 AS c3, + b = 3 AND a = 1 AS c4, + a = 2 OR b = 4 AS c5, + a = 2 OR b = 5 AS c6, + b = 6 OR a = 3 AS c7, + b = 7 OR a = 3 AS c8, + CASE WHEN (a = 4) THEN 0 ELSE 1 END AS c9, + CASE WHEN (a = 4) THEN 0 ELSE 2 END AS c10, + CASE WHEN (b = 8) THEN a + 1 ELSE 0 END AS c11, + CASE WHEN (b = 9) THEN a + 1 ELSE 0 END AS c12, + CASE WHEN (b = 10) THEN 0 ELSE a + 2 END AS c13, + CASE WHEN (b = 11) THEN 0 ELSE a + 2 END AS c14 +FROM t1 +---- +logical_plan +01)Projection: t1.a = Float64(0) AND t1.b = Float64(0) AS c1, t1.a = Float64(0) AND t1.b = Float64(1) AS c2, t1.b = Float64(2) AND t1.a = Float64(1) AS c3, t1.b = Float64(3) AND t1.a = Float64(1) AS c4, t1.a = Float64(2) OR t1.b = Float64(4) AS c5, t1.a = Float64(2) OR t1.b = Float64(5) AS c6, t1.b = Float64(6) OR t1.a = Float64(3) AS c7, t1.b = Float64(7) OR t1.a = Float64(3) AS c8, CASE WHEN t1.a = Float64(4) THEN Int64(0) ELSE Int64(1) END AS c9, CASE WHEN t1.a = Float64(4) THEN Int64(0) ELSE Int64(2) END AS c10, CASE WHEN t1.b = Float64(8) THEN t1.a + Float64(1) ELSE Float64(0) END AS c11, CASE WHEN t1.b = Float64(9) THEN t1.a + Float64(1) ELSE Float64(0) END AS c12, CASE WHEN t1.b = Float64(10) THEN Float64(0) ELSE t1.a + Float64(2) END AS c13, CASE WHEN t1.b = Float64(11) THEN Float64(0) ELSE t1.a + Float64(2) END AS c14 +02)--TableScan: t1 projection=[a, b] +physical_plan +01)ProjectionExec: expr=[a@0 = 0 AND b@1 = 0 as c1, a@0 = 0 AND b@1 = 1 as c2, b@1 = 2 AND a@0 = 1 as c3, b@1 = 3 AND a@0 = 1 as c4, a@0 = 2 OR b@1 = 4 as c5, a@0 = 2 OR b@1 = 5 as c6, b@1 = 6 OR a@0 = 3 as c7, b@1 = 7 OR a@0 = 3 as c8, CASE WHEN a@0 = 4 THEN 0 ELSE 1 END as c9, CASE WHEN a@0 = 4 THEN 0 ELSE 2 END as c10, CASE WHEN b@1 = 8 THEN a@0 + 1 ELSE 0 END as c11, CASE WHEN b@1 = 9 THEN a@0 + 1 ELSE 0 END as c12, CASE WHEN b@1 = 10 THEN 0 ELSE a@0 + 2 END as c13, CASE WHEN b@1 = 11 THEN 0 ELSE a@0 + 2 END as c14] +02)--MemoryExec: partitions=1, partition_sizes=[0] + +# Common children of volatile, short-circuit expression +# TODO: consider surely executed children of "short circuited"s for CSE. i.e. `a = 0`, `a = 2`, `a = 4` should be extracted +query TT +EXPLAIN SELECT + a = 0 AND b = random() AS c1, + a = 0 AND b = 1 + random() AS c2, + b = 2 + random() AND a = 1 AS c3, + b = 3 + random() AND a = 1 AS c4, + a = 2 OR b = 4 + random() AS c5, + a = 2 OR b = 5 + random() AS c6, + b = 6 + random() OR a = 3 AS c7, + b = 7 + random() OR a = 3 AS c8, + CASE WHEN (a = 4) THEN random() ELSE 1 END AS c9, + CASE WHEN (a = 4) THEN random() ELSE 2 END AS c10, + CASE WHEN (b = 8 + random()) THEN a + 1 ELSE 0 END AS c11, + CASE WHEN (b = 9 + random()) THEN a + 1 ELSE 0 END AS c12, + CASE WHEN (b = 10 + random()) THEN 0 ELSE a + 2 END AS c13, + CASE WHEN (b = 11 + random()) THEN 0 ELSE a + 2 END AS c14 +FROM t1 +---- +logical_plan +01)Projection: t1.a = Float64(0) AND t1.b = random() AS c1, t1.a = Float64(0) AND t1.b = Float64(1) + random() AS c2, t1.b = Float64(2) + random() AND t1.a = Float64(1) AS c3, t1.b = Float64(3) + random() AND t1.a = Float64(1) AS c4, t1.a = Float64(2) OR t1.b = Float64(4) + random() AS c5, t1.a = Float64(2) OR t1.b = Float64(5) + random() AS c6, t1.b = Float64(6) + random() OR t1.a = Float64(3) AS c7, t1.b = Float64(7) + random() OR t1.a = Float64(3) AS c8, CASE WHEN t1.a = Float64(4) THEN random() ELSE Float64(1) END AS c9, CASE WHEN t1.a = Float64(4) THEN random() ELSE Float64(2) END AS c10, CASE WHEN t1.b = Float64(8) + random() THEN t1.a + Float64(1) ELSE Float64(0) END AS c11, CASE WHEN t1.b = Float64(9) + random() THEN t1.a + Float64(1) ELSE Float64(0) END AS c12, CASE WHEN t1.b = Float64(10) + random() THEN Float64(0) ELSE t1.a + Float64(2) END AS c13, CASE WHEN t1.b = Float64(11) + random() THEN Float64(0) ELSE t1.a + Float64(2) END AS c14 +02)--TableScan: t1 projection=[a, b] +physical_plan +01)ProjectionExec: expr=[a@0 = 0 AND b@1 = random() as c1, a@0 = 0 AND b@1 = 1 + random() as c2, b@1 = 2 + random() AND a@0 = 1 as c3, b@1 = 3 + random() AND a@0 = 1 as c4, a@0 = 2 OR b@1 = 4 + random() as c5, a@0 = 2 OR b@1 = 5 + random() as c6, b@1 = 6 + random() OR a@0 = 3 as c7, b@1 = 7 + random() OR a@0 = 3 as c8, CASE WHEN a@0 = 4 THEN random() ELSE 1 END as c9, CASE WHEN a@0 = 4 THEN random() ELSE 2 END as c10, CASE WHEN b@1 = 8 + random() THEN a@0 + 1 ELSE 0 END as c11, CASE WHEN b@1 = 9 + random() THEN a@0 + 1 ELSE 0 END as c12, CASE WHEN b@1 = 10 + random() THEN 0 ELSE a@0 + 2 END as c13, CASE WHEN b@1 = 11 + random() THEN 0 ELSE a@0 + 2 END as c14] +02)--MemoryExec: partitions=1, partition_sizes=[0] + +# Common volatile children of short-circuit expression +query TT +EXPLAIN SELECT + a = random() AND b = 0 AS c1, + a = random() AND b = 1 AS c2, + a = 2 + random() OR b = 4 AS c3, + a = 2 + random() OR b = 5 AS c4, + CASE WHEN (a = 4 + random()) THEN 0 ELSE 1 END AS c5, + CASE WHEN (a = 4 + random()) THEN 0 ELSE 2 END AS c6 +FROM t1 +---- +logical_plan +01)Projection: t1.a = random() AND t1.b = Float64(0) AS c1, t1.a = random() AND t1.b = Float64(1) AS c2, t1.a = Float64(2) + random() OR t1.b = Float64(4) AS c3, t1.a = Float64(2) + random() OR t1.b = Float64(5) AS c4, CASE WHEN t1.a = Float64(4) + random() THEN Int64(0) ELSE Int64(1) END AS c5, CASE WHEN t1.a = Float64(4) + random() THEN Int64(0) ELSE Int64(2) END AS c6 +02)--TableScan: t1 projection=[a, b] +physical_plan +01)ProjectionExec: expr=[a@0 = random() AND b@1 = 0 as c1, a@0 = random() AND b@1 = 1 as c2, a@0 = 2 + random() OR b@1 = 4 as c3, a@0 = 2 + random() OR b@1 = 5 as c4, CASE WHEN a@0 = 4 + random() THEN 0 ELSE 1 END as c5, CASE WHEN a@0 = 4 + random() THEN 0 ELSE 2 END as c6] +02)--MemoryExec: partitions=1, partition_sizes=[0] From 21917ed8db35155b5a6b3a3bd62de4bcc1adf994 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 6 Jul 2024 07:24:42 -0400 Subject: [PATCH 5/6] Avoid adding datafusion function dependency --- datafusion/optimizer/Cargo.toml | 1 - .../optimizer/src/common_subexpr_eliminate.rs | 56 +++++++++++++++++-- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index 54d8c472f13f..1a9e9630c076 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -45,7 +45,6 @@ async-trait = { workspace = true } chrono = { workspace = true } datafusion-common = { workspace = true, default-features = true } datafusion-expr = { workspace = true } -datafusion-functions = { workspace = true } datafusion-physical-expr = { workspace = true } hashbrown = { workspace = true } indexmap = { workspace = true } diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs index d03cc361b9bc..4a4933fe9cfd 100644 --- a/datafusion/optimizer/src/common_subexpr_eliminate.rs +++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs @@ -1127,18 +1127,19 @@ fn replace_common_expr<'n>( #[cfg(test)] mod test { + use std::any::Any; use std::collections::HashSet; use std::iter; use arrow::datatypes::{DataType, Field, Schema}; - use datafusion_expr::expr::{AggregateFunction, ScalarFunction}; + use datafusion_expr::expr::AggregateFunction; use datafusion_expr::logical_plan::{table_scan, JoinType}; use datafusion_expr::{ - grouping_set, AccumulatorFactoryFunction, AggregateUDF, BinaryExpr, Signature, - SimpleAggregateUDF, Volatility, + grouping_set, AccumulatorFactoryFunction, AggregateUDF, BinaryExpr, + ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, SimpleAggregateUDF, + Volatility, }; use datafusion_expr::{lit, logical_plan::builder::LogicalPlanBuilder}; - use datafusion_functions::math; use crate::optimizer::OptimizerContext; use crate::test::*; @@ -1871,7 +1872,7 @@ mod test { let table_scan = test_table_scan()?; let extracted_child = col("a") + col("b"); - let rand = Expr::ScalarFunction(ScalarFunction::new_udf(math::random(), vec![])); + let rand = rand_func().call(vec![]); let not_extracted_volatile = extracted_child + rand; let plan = LogicalPlanBuilder::from(table_scan.clone()) .project(vec![ @@ -1893,7 +1894,7 @@ mod test { fn test_volatile_short_circuits() -> Result<()> { let table_scan = test_table_scan()?; - let rand = Expr::ScalarFunction(ScalarFunction::new_udf(math::random(), vec![])); + let rand = rand_func().call(vec![]); let not_extracted_volatile_short_circuit_2 = rand.clone().eq(lit(0)).or(col("b").eq(lit(0))); let not_extracted_volatile_short_circuit_1 = @@ -1914,4 +1915,47 @@ mod test { Ok(()) } + + /// returns a "random" function that is marked volatile (aka each invocation + /// returns a different value) + /// + /// Does not use datafusion_functions::rand to avoid introducing a + /// dependency on that crate. + fn rand_func() -> ScalarUDF { + ScalarUDF::new_from_impl(RandomStub::new()) + } + + #[derive(Debug)] + struct RandomStub { + signature: Signature, + } + + impl RandomStub { + fn new() -> Self { + Self { + signature: Signature::exact(vec![], Volatility::Volatile), + } + } + } + impl ScalarUDFImpl for RandomStub { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "random" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn invoke(&self, _args: &[ColumnarValue]) -> Result { + unimplemented!() + } + } } From 0f3932ddc90f6d7c4f0c0c4d30c71c06db5c6561 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 6 Jul 2024 07:51:03 -0400 Subject: [PATCH 6/6] revert changes to datafusion-cli.lock --- datafusion-cli/Cargo.lock | 103 +++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 380723bd72a5..5fc8dbcfdfb3 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -875,9 +875,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.104" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +checksum = "2755ff20a1d93490d26ba33a6f092a38a508398a5320df5d4b3014fcccce9410" dependencies = [ "jobserver", "libc", @@ -900,7 +900,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.6", + "windows-targets 0.52.5", ] [[package]] @@ -1333,7 +1333,6 @@ dependencies = [ "chrono", "datafusion-common", "datafusion-expr", - "datafusion-functions", "datafusion-physical-expr", "hashbrown 0.14.5", "indexmap 2.2.6", @@ -1965,9 +1964,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", @@ -2006,10 +2005,10 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.3.1", "hyper-util", "rustls 0.23.10", - "rustls-native-certs 0.7.1", + "rustls-native-certs 0.7.0", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -2018,16 +2017,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.3.1", "pin-project-lite", "socket2", "tokio", @@ -2502,7 +2501,7 @@ dependencies = [ "chrono", "futures", "humantime", - "hyper 1.4.0", + "hyper 1.3.1", "itertools", "md-5", "parking_lot", @@ -2574,7 +2573,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.6", + "windows-targets 0.52.5", ] [[package]] @@ -2976,7 +2975,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.3.1", "hyper-rustls 0.27.2", "hyper-util", "ipnet", @@ -2988,7 +2987,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.10", - "rustls-native-certs 0.7.1", + "rustls-native-certs 0.7.0", "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", @@ -3143,9 +3142,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", "rustls-pemfile 2.1.2", @@ -3181,9 +3180,9 @@ checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" -version = "0.102.5" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -3316,9 +3315,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "e8eddb61f0697cc3989c5d64b452f5488e2b8a60fd7d5076a3045076ffef8cb0" dependencies = [ "itoa", "ryu", @@ -4098,7 +4097,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.52.5", ] [[package]] @@ -4116,7 +4115,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.52.5", ] [[package]] @@ -4136,18 +4135,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -4158,9 +4157,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -4170,9 +4169,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -4182,15 +4181,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" [[package]] name = "windows_i686_gnullvm" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -4200,9 +4199,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -4212,9 +4211,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -4224,9 +4223,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -4236,9 +4235,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winreg" @@ -4267,18 +4266,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.35" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote",