From 91c159c43854af11696363f060f6492410595af7 Mon Sep 17 00:00:00 2001 From: Michael-J-Ward Date: Sat, 10 Aug 2024 15:51:01 -0500 Subject: [PATCH] migrate AggregateExt to ExprFunctionExt Also removed `sqlparser` dependency since it's re-exported upstream. Ref: https://github.com/apache/datafusion/pull/11550 --- Cargo.lock | 20 +++++--------------- Cargo.toml | 1 - src/common/data_type.rs | 18 ++++++++++-------- src/functions.rs | 12 +++++------- 4 files changed, 20 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee3b3c43..6ed7f4f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,7 +791,7 @@ dependencies = [ "paste", "pin-project-lite", "rand", - "sqlparser 0.49.0", + "sqlparser", "tempfile", "tokio", "tokio-util", @@ -834,7 +834,7 @@ dependencies = [ "object_store", "parquet", "pyo3", - "sqlparser 0.49.0", + "sqlparser", ] [[package]] @@ -881,7 +881,7 @@ dependencies = [ "datafusion-physical-expr-common", "paste", "serde_json", - "sqlparser 0.49.0", + "sqlparser", "strum 0.26.3", "strum_macros 0.26.4", ] @@ -938,7 +938,7 @@ dependencies = [ "datafusion-physical-expr-common", "log", "paste", - "sqlparser 0.49.0", + "sqlparser", ] [[package]] @@ -1122,7 +1122,6 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "sqlparser 0.47.0", "syn 2.0.72", "tokio", "url", @@ -1141,7 +1140,7 @@ dependencies = [ "datafusion-expr", "log", "regex", - "sqlparser 0.49.0", + "sqlparser", "strum 0.26.3", ] @@ -2965,15 +2964,6 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -[[package]] -name = "sqlparser" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" -dependencies = [ - "log", -] - [[package]] name = "sqlparser" version = "0.49.0" diff --git a/Cargo.toml b/Cargo.toml index 240704fa..d6f65edd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,6 @@ parking_lot = "0.12" regex-syntax = "0.8" syn = "2.0.68" url = "2" -sqlparser = "0.47.0" [build-dependencies] pyo3-build-config = "0.21" diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 469bb789..ffc8b843 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -18,6 +18,7 @@ use datafusion::arrow::array::Array; use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; use datafusion_common::{DataFusionError, ScalarValue}; +use datafusion_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use pyo3::{exceptions::PyValueError, prelude::*}; use crate::errors::py_datafusion_err; @@ -775,20 +776,21 @@ pub enum NullTreatment { RESPECT_NULLS, } -impl From for sqlparser::ast::NullTreatment { - fn from(null_treatment: NullTreatment) -> sqlparser::ast::NullTreatment { + +impl From for DFNullTreatment { + fn from(null_treatment: NullTreatment) -> DFNullTreatment { match null_treatment { - NullTreatment::IGNORE_NULLS => sqlparser::ast::NullTreatment::IgnoreNulls, - NullTreatment::RESPECT_NULLS => sqlparser::ast::NullTreatment::RespectNulls, + NullTreatment::IGNORE_NULLS => DFNullTreatment::IgnoreNulls, + NullTreatment::RESPECT_NULLS => DFNullTreatment::RespectNulls, } } } -impl From for NullTreatment { - fn from(null_treatment: sqlparser::ast::NullTreatment) -> NullTreatment { +impl From for NullTreatment { + fn from(null_treatment: DFNullTreatment) -> NullTreatment { match null_treatment { - sqlparser::ast::NullTreatment::IgnoreNulls => NullTreatment::IGNORE_NULLS, - sqlparser::ast::NullTreatment::RespectNulls => NullTreatment::RESPECT_NULLS, + DFNullTreatment::IgnoreNulls => NullTreatment::IGNORE_NULLS, + DFNullTreatment::RespectNulls => NullTreatment::RESPECT_NULLS, } } } diff --git a/src/functions.rs b/src/functions.rs index dea4d198..4db3a3ea 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::functions_aggregate::all_default_aggregate_functions; -use datafusion_expr::ExprFunctionExt as AggregateExt; +use datafusion_expr::ExprFunctionExt; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common::data_type::NullTreatment; @@ -30,6 +30,7 @@ use datafusion::functions; use datafusion::functions_aggregate; use datafusion_common::{Column, ScalarValue, TableReference}; use datafusion_expr::expr::Alias; +use datafusion_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use datafusion_expr::{ expr::{find_df_window_func, AggregateFunction, Sort, WindowFunction}, lit, Expr, WindowFunctionDefinition, @@ -340,9 +341,8 @@ pub fn first_value( builder = builder.filter(filter.expr); } - if let Some(null_treatment) = null_treatment { - builder = builder.null_treatment(null_treatment.into()) - } + // would be nice if all the options builder methods accepted Option ... + builder = builder.null_treatment(null_treatment.map(DFNullTreatment::from)); Ok(builder.build()?.into()) } @@ -371,9 +371,7 @@ pub fn last_value( builder = builder.filter(filter.expr); } - if let Some(null_treatment) = null_treatment { - builder = builder.null_treatment(null_treatment.into()) - } + builder = builder.null_treatment(null_treatment.map(DFNullTreatment::from)); Ok(builder.build()?.into()) }