From 0bc07c628cc8c3eea0945877f03b77f72ab48e00 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 22 Feb 2023 19:05:13 -0500 Subject: [PATCH 1/4] Add bool_expr --- src/expr.rs | 13 ++ src/expr/bool_expr.rs | 300 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 src/expr/bool_expr.rs diff --git a/src/expr.rs b/src/expr.rs index 90ce6bf0e..f1827d1f7 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -29,10 +29,13 @@ use crate::expr::column::PyColumn; use crate::expr::literal::PyLiteral; use datafusion::scalar::ScalarValue; +use self::bool_expr::{PyNot, PyIsNotNull, PyIsNull, PyIsTrue, PyIsFalse, PyIsUnknown, PyIsNotTrue, PyIsNotFalse, PyIsNotUnknown, PyNegative}; + pub mod aggregate; pub mod aggregate_expr; pub mod analyze; pub mod binary_expr; +pub mod bool_expr; pub mod column; pub mod filter; pub mod limit; @@ -177,6 +180,16 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; // operators m.add_class::()?; m.add_class::()?; diff --git a/src/expr/bool_expr.rs b/src/expr/bool_expr.rs new file mode 100644 index 000000000..b22b81b88 --- /dev/null +++ b/src/expr/bool_expr.rs @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion_expr::Expr; +use pyo3::prelude::*; +use std::fmt::{self, Display, Formatter}; + + +#[pyclass(name = "Not", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyNot { + expr: Expr, +} + +impl PyNot { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + +#[pyclass(name = "IsNotNull", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyIsNotNull { + expr: Expr, +} + +impl PyIsNotNull { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + +#[pyclass(name = "IsNull", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyIsNull { + expr: Expr, +} + +impl PyIsNull { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + +#[pyclass(name = "IsTrue", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyIsTrue { + expr: Expr, +} + +impl PyIsTrue { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + +#[pyclass(name = "IsFalse", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyIsFalse { + expr: Expr, +} + +impl PyIsFalse { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + +#[pyclass(name = "IsUnknown", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyIsUnknown { + expr: Expr, +} + +impl PyIsUnknown { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + +#[pyclass(name = "IsNotTrue", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyIsNotTrue { + expr: Expr, +} + +impl PyIsNotTrue { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + +#[pyclass(name = "IsNotFalse", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyIsNotFalse { + expr: Expr, +} + +impl PyIsNotFalse { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + + +#[pyclass(name = "IsNotUnknown", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyIsNotUnknown { + expr: Expr, +} + +impl PyIsNotUnknown { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + +#[pyclass(name = "Negative", module = "datafusion.expr", subclass)] +#[derive(Clone, Debug)] +pub struct PyNegative { + expr: Expr, +} + +impl PyNegative { + pub fn new(expr: Expr) -> Self { + Self { expr } + } +} + + + +// impl Display for PyLike { +// fn fmt(&self, f: &mut Formatter) -> fmt::Result { +// write!( +// f, +// "Like +// Negated: {:?} +// Expr: {:?} +// Pattern: {:?} +// Escape_Char: {:?}", +// &self.negated(), +// &self.expr(), +// &self.pattern(), +// &self.escape_char() +// ) +// } +// } + +// #[pymethods] +// impl PyLike { +// fn negated(&self) -> PyResult { +// Ok(self.like.negated) +// } + +// fn expr(&self) -> PyResult { +// Ok((*self.like.expr).clone().into()) +// } + +// fn pattern(&self) -> PyResult { +// Ok((*self.like.pattern).clone().into()) +// } + +// fn escape_char(&self) -> PyResult> { +// Ok(self.like.escape_char) +// } + +// fn __repr__(&self) -> String { +// format!("Like({})", self) +// } +// } + +// #[pyclass(name = "ILike", module = "datafusion.expr", subclass)] +// #[derive(Clone)] +// pub struct PyILike { +// like: Like, +// } + +// impl From for PyILike { +// fn from(like: Like) -> PyILike { +// PyILike { like } +// } +// } + +// impl From for Like { +// fn from(like: PyILike) -> Self { +// like.like +// } +// } + +// impl Display for PyILike { +// fn fmt(&self, f: &mut Formatter) -> fmt::Result { +// write!( +// f, +// "ILike +// Negated: {:?} +// Expr: {:?} +// Pattern: {:?} +// Escape_Char: {:?}", +// &self.negated(), +// &self.expr(), +// &self.pattern(), +// &self.escape_char() +// ) +// } +// } + +// #[pymethods] +// impl PyILike { +// fn negated(&self) -> PyResult { +// Ok(self.like.negated) +// } + +// fn expr(&self) -> PyResult { +// Ok((*self.like.expr).clone().into()) +// } + +// fn pattern(&self) -> PyResult { +// Ok((*self.like.pattern).clone().into()) +// } + +// fn escape_char(&self) -> PyResult> { +// Ok(self.like.escape_char) +// } + +// fn __repr__(&self) -> String { +// format!("Like({})", self) +// } +// } + +// #[pyclass(name = "SimilarTo", module = "datafusion.expr", subclass)] +// #[derive(Clone)] +// pub struct PySimilarTo { +// like: Like, +// } + +// impl From for PySimilarTo { +// fn from(like: Like) -> PySimilarTo { +// PySimilarTo { like } +// } +// } + +// impl From for Like { +// fn from(like: PySimilarTo) -> Self { +// like.like +// } +// } + +// impl Display for PySimilarTo { +// fn fmt(&self, f: &mut Formatter) -> fmt::Result { +// write!( +// f, +// "SimilarTo +// Negated: {:?} +// Expr: {:?} +// Pattern: {:?} +// Escape_Char: {:?}", +// &self.negated(), +// &self.expr(), +// &self.pattern(), +// &self.escape_char() +// ) +// } +// } + +// #[pymethods] +// impl PySimilarTo { +// fn negated(&self) -> PyResult { +// Ok(self.like.negated) +// } + +// fn expr(&self) -> PyResult { +// Ok((*self.like.expr).clone().into()) +// } + +// fn pattern(&self) -> PyResult { +// Ok((*self.like.pattern).clone().into()) +// } + +// fn escape_char(&self) -> PyResult> { +// Ok(self.like.escape_char) +// } + +// fn __repr__(&self) -> String { +// format!("Like({})", self) +// } +// } From 6cf2fb3030676c7c1af4953d51c60b23f0a3cf3c Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 22 Feb 2023 21:07:02 -0500 Subject: [PATCH 2/4] boolean expression bindings --- src/expr.rs | 15 +- src/expr/bool_expr.rs | 334 ++++++++++++++++++++++-------------------- 2 files changed, 192 insertions(+), 157 deletions(-) diff --git a/src/expr.rs b/src/expr.rs index f1827d1f7..4c471ab3b 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -29,7 +29,10 @@ use crate::expr::column::PyColumn; use crate::expr::literal::PyLiteral; use datafusion::scalar::ScalarValue; -use self::bool_expr::{PyNot, PyIsNotNull, PyIsNull, PyIsTrue, PyIsFalse, PyIsUnknown, PyIsNotTrue, PyIsNotFalse, PyIsNotUnknown, PyNegative}; +use self::bool_expr::{ + PyIsFalse, PyIsNotFalse, PyIsNotNull, PyIsNotTrue, PyIsNotUnknown, PyIsNull, PyIsTrue, + PyIsUnknown, PyNegative, PyNot, +}; pub mod aggregate; pub mod aggregate_expr; @@ -72,6 +75,16 @@ impl PyExpr { Expr::Column(col) => Ok(PyColumn::from(col.clone()).into_py(py)), Expr::Literal(value) => Ok(PyLiteral::from(value.clone()).into_py(py)), Expr::BinaryExpr(expr) => Ok(PyBinaryExpr::from(expr.clone()).into_py(py)), + Expr::Not(expr) => Ok(PyNot::new(*expr.clone()).into_py(py)), + Expr::IsNotNull(expr) => Ok(PyIsNotNull::new(*expr.clone()).into_py(py)), + Expr::IsNull(expr) => Ok(PyIsNull::new(*expr.clone()).into_py(py)), + Expr::IsTrue(expr) => Ok(PyIsTrue::new(*expr.clone()).into_py(py)), + Expr::IsFalse(expr) => Ok(PyIsFalse::new(*expr.clone()).into_py(py)), + Expr::IsUnknown(expr) => Ok(PyIsUnknown::new(*expr.clone()).into_py(py)), + Expr::IsNotTrue(expr) => Ok(PyIsNotTrue::new(*expr.clone()).into_py(py)), + Expr::IsNotFalse(expr) => Ok(PyIsNotFalse::new(*expr.clone()).into_py(py)), + Expr::IsNotUnknown(expr) => Ok(PyIsNotUnknown::new(*expr.clone()).into_py(py)), + Expr::Negative(expr) => Ok(PyNegative::new(*expr.clone()).into_py(py)), Expr::AggregateFunction(expr) => { Ok(PyAggregateFunction::from(expr.clone()).into_py(py)) } diff --git a/src/expr/bool_expr.rs b/src/expr/bool_expr.rs index b22b81b88..d1502a4eb 100644 --- a/src/expr/bool_expr.rs +++ b/src/expr/bool_expr.rs @@ -19,6 +19,7 @@ use datafusion_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; +use super::PyExpr; #[pyclass(name = "Not", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] @@ -32,6 +33,24 @@ impl PyNot { } } +impl Display for PyNot { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "Not + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyNot { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} + #[pyclass(name = "IsNotNull", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] pub struct PyIsNotNull { @@ -44,6 +63,24 @@ impl PyIsNotNull { } } +impl Display for PyIsNotNull { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "IsNotNull + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyIsNotNull { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} + #[pyclass(name = "IsNull", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] pub struct PyIsNull { @@ -56,6 +93,24 @@ impl PyIsNull { } } +impl Display for PyIsNull { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "IsNull + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyIsNull { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} + #[pyclass(name = "IsTrue", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] pub struct PyIsTrue { @@ -68,6 +123,24 @@ impl PyIsTrue { } } +impl Display for PyIsTrue { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "IsTrue + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyIsTrue { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} + #[pyclass(name = "IsFalse", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] pub struct PyIsFalse { @@ -80,6 +153,24 @@ impl PyIsFalse { } } +impl Display for PyIsFalse { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "IsFalse + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyIsFalse { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} + #[pyclass(name = "IsUnknown", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] pub struct PyIsUnknown { @@ -92,6 +183,24 @@ impl PyIsUnknown { } } +impl Display for PyIsUnknown { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "IsUnknown + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyIsUnknown { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} + #[pyclass(name = "IsNotTrue", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] pub struct PyIsNotTrue { @@ -104,6 +213,24 @@ impl PyIsNotTrue { } } +impl Display for PyIsNotTrue { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "IsNotTrue + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyIsNotTrue { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} + #[pyclass(name = "IsNotFalse", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] pub struct PyIsNotFalse { @@ -116,6 +243,23 @@ impl PyIsNotFalse { } } +impl Display for PyIsNotFalse { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "IsNotFalse + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyIsNotFalse { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} #[pyclass(name = "IsNotUnknown", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] @@ -129,6 +273,24 @@ impl PyIsNotUnknown { } } +impl Display for PyIsNotUnknown { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "IsNotUnknown + Expr: {}", + &self.expr + ) + } +} + +#[pymethods] +impl PyIsNotUnknown { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} + #[pyclass(name = "Negative", module = "datafusion.expr", subclass)] #[derive(Clone, Debug)] pub struct PyNegative { @@ -141,160 +303,20 @@ impl PyNegative { } } +impl Display for PyNegative { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "Negative + Expr: {}", + &self.expr + ) + } +} - -// impl Display for PyLike { -// fn fmt(&self, f: &mut Formatter) -> fmt::Result { -// write!( -// f, -// "Like -// Negated: {:?} -// Expr: {:?} -// Pattern: {:?} -// Escape_Char: {:?}", -// &self.negated(), -// &self.expr(), -// &self.pattern(), -// &self.escape_char() -// ) -// } -// } - -// #[pymethods] -// impl PyLike { -// fn negated(&self) -> PyResult { -// Ok(self.like.negated) -// } - -// fn expr(&self) -> PyResult { -// Ok((*self.like.expr).clone().into()) -// } - -// fn pattern(&self) -> PyResult { -// Ok((*self.like.pattern).clone().into()) -// } - -// fn escape_char(&self) -> PyResult> { -// Ok(self.like.escape_char) -// } - -// fn __repr__(&self) -> String { -// format!("Like({})", self) -// } -// } - -// #[pyclass(name = "ILike", module = "datafusion.expr", subclass)] -// #[derive(Clone)] -// pub struct PyILike { -// like: Like, -// } - -// impl From for PyILike { -// fn from(like: Like) -> PyILike { -// PyILike { like } -// } -// } - -// impl From for Like { -// fn from(like: PyILike) -> Self { -// like.like -// } -// } - -// impl Display for PyILike { -// fn fmt(&self, f: &mut Formatter) -> fmt::Result { -// write!( -// f, -// "ILike -// Negated: {:?} -// Expr: {:?} -// Pattern: {:?} -// Escape_Char: {:?}", -// &self.negated(), -// &self.expr(), -// &self.pattern(), -// &self.escape_char() -// ) -// } -// } - -// #[pymethods] -// impl PyILike { -// fn negated(&self) -> PyResult { -// Ok(self.like.negated) -// } - -// fn expr(&self) -> PyResult { -// Ok((*self.like.expr).clone().into()) -// } - -// fn pattern(&self) -> PyResult { -// Ok((*self.like.pattern).clone().into()) -// } - -// fn escape_char(&self) -> PyResult> { -// Ok(self.like.escape_char) -// } - -// fn __repr__(&self) -> String { -// format!("Like({})", self) -// } -// } - -// #[pyclass(name = "SimilarTo", module = "datafusion.expr", subclass)] -// #[derive(Clone)] -// pub struct PySimilarTo { -// like: Like, -// } - -// impl From for PySimilarTo { -// fn from(like: Like) -> PySimilarTo { -// PySimilarTo { like } -// } -// } - -// impl From for Like { -// fn from(like: PySimilarTo) -> Self { -// like.like -// } -// } - -// impl Display for PySimilarTo { -// fn fmt(&self, f: &mut Formatter) -> fmt::Result { -// write!( -// f, -// "SimilarTo -// Negated: {:?} -// Expr: {:?} -// Pattern: {:?} -// Escape_Char: {:?}", -// &self.negated(), -// &self.expr(), -// &self.pattern(), -// &self.escape_char() -// ) -// } -// } - -// #[pymethods] -// impl PySimilarTo { -// fn negated(&self) -> PyResult { -// Ok(self.like.negated) -// } - -// fn expr(&self) -> PyResult { -// Ok((*self.like.expr).clone().into()) -// } - -// fn pattern(&self) -> PyResult { -// Ok((*self.like.pattern).clone().into()) -// } - -// fn escape_char(&self) -> PyResult> { -// Ok(self.like.escape_char) -// } - -// fn __repr__(&self) -> String { -// format!("Like({})", self) -// } -// } +#[pymethods] +impl PyNegative { + fn expr(&self) -> PyResult { + Ok(self.expr.clone().into()) + } +} From 00feab1cd5ad2c52175fba183d4ef7095d6785bd Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 22 Feb 2023 21:14:07 -0500 Subject: [PATCH 3/4] pytest for importing boolean expr structs --- datafusion/__init__.py | 18 ++++++++++++++++++ datafusion/tests/test_imports.py | 18 ++++++++++++++++++ src/expr.rs | 3 --- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/datafusion/__init__.py b/datafusion/__init__.py index 8ce63018d..2529d64fc 100644 --- a/datafusion/__init__.py +++ b/datafusion/__init__.py @@ -50,6 +50,15 @@ ScalarVariable, Sort, TableScan, + Not, + IsNotNull, + IsTrue, + IsFalse, + IsUnknown, + IsNotTrue, + IsNotFalse, + IsNotUnknown, + Negative, ) __version__ = importlib_metadata.version(__name__) @@ -75,6 +84,15 @@ "Filter", "ScalarVariable", "Alias", + "Not", + "IsNotNull", + "IsTrue", + "IsFalse", + "IsUnknown", + "IsNotTrue", + "IsNotFalse", + "IsNotUnknown", + "Negative", ] diff --git a/datafusion/tests/test_imports.py b/datafusion/tests/test_imports.py index 83d64f385..e50276063 100644 --- a/datafusion/tests/test_imports.py +++ b/datafusion/tests/test_imports.py @@ -46,6 +46,15 @@ Analyze, ScalarVariable, Alias, + Not, + IsNotNull, + IsTrue, + IsFalse, + IsUnknown, + IsNotTrue, + IsNotFalse, + IsNotUnknown, + Negative, ) @@ -81,6 +90,15 @@ def test_class_module_is_datafusion(): Analyze, ScalarVariable, Alias, + Not, + IsNotNull, + IsTrue, + IsFalse, + IsUnknown, + IsNotTrue, + IsNotFalse, + IsNotUnknown, + Negative, ]: assert klass.__module__ == "datafusion.expr" diff --git a/src/expr.rs b/src/expr.rs index dacad0aaf..621093ae5 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -213,7 +213,6 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; -<<<<<<< HEAD m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -224,10 +223,8 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; -======= m.add_class::()?; m.add_class::()?; ->>>>>>> upstream/main // operators m.add_class::()?; m.add_class::()?; From 8b214ca1f8df96cbd719ea4d6ef173bed4b49a96 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 22 Feb 2023 21:15:41 -0500 Subject: [PATCH 4/4] cargo fmt --- src/expr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/expr.rs b/src/expr.rs index 621093ae5..2e2d8039f 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -29,11 +29,11 @@ use crate::expr::column::PyColumn; use crate::expr::literal::PyLiteral; use datafusion::scalar::ScalarValue; +use self::alias::PyAlias; use self::bool_expr::{ PyIsFalse, PyIsNotFalse, PyIsNotNull, PyIsNotTrue, PyIsNotUnknown, PyIsNull, PyIsTrue, PyIsUnknown, PyNegative, PyNot, }; -use self::alias::PyAlias; use self::scalar_variable::PyScalarVariable; pub mod aggregate;