Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for filter pushdown rule #924

Merged
merged 21 commits into from
Dec 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
9fb97f3
Enable inexact filters for predicate pushdown, add helper to get fitl…
ayushdg Nov 16, 2022
0683336
Update table scan logic to add filters
ayushdg Nov 16, 2022
9a9fa1e
Update PyTableScan to include input schema
ayushdg Nov 18, 2022
8d1fd4a
Update DaskTableSource to allow filtering on all expr's
ayushdg Nov 18, 2022
6b0b544
Change order to apply filters before projections
ayushdg Nov 18, 2022
9e962dd
Merge branch 'main' of github.com:dask-contrib/dask-sql into support-…
ayushdg Nov 18, 2022
5a4b291
Clean up filter conjuction application
ayushdg Nov 28, 2022
6df4bd0
use filter_pushdown_rule from datafusion
ayushdg Nov 28, 2022
1964368
Update predicate pushdown tests
ayushdg Nov 28, 2022
49e90d6
Update predicate pushdown tests
ayushdg Nov 28, 2022
8fa35de
Merge branch 'main' into support-filter-pushdown-rule
ayushdg Nov 28, 2022
294bda1
unxfail q21
ayushdg Nov 29, 2022
356d064
Merge branch 'support-filter-pushdown-rule' of github.com:ayushdg/das…
ayushdg Nov 29, 2022
8e06b46
Merge branch 'main' of github.com:dask-contrib/dask-sql into support-…
ayushdg Nov 30, 2022
cfe2905
Update DaskTableSource filterPushDown comments
ayushdg Nov 30, 2022
3b85ac4
Reenable clippy check for supports_filter_pushdown
ayushdg Nov 30, 2022
521129c
Simplify apply_filter conditional check
ayushdg Nov 30, 2022
cdeeb49
Merge branch 'main' into support-filter-pushdown-rule
ayushdg Nov 30, 2022
acdab19
Merge branch 'main' into support-filter-pushdown-rule
charlesbluca Dec 1, 2022
47eee60
Un-xfail q40
charlesbluca Dec 1, 2022
2090f86
Rerun tests
ayushdg Dec 1, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions dask_planner/src/sql/logical/table_scan.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
use datafusion_expr::logical_plan::TableScan;
use std::sync::Arc;

use datafusion_common::DFSchema;
use datafusion_expr::{logical_plan::TableScan, LogicalPlan};
use pyo3::prelude::*;

use crate::sql::{exceptions::py_type_err, logical};
use crate::{
expression::{py_expr_list, PyExpr},
sql::exceptions::py_type_err,
};

#[pyclass(name = "TableScan", module = "dask_planner", subclass)]
#[derive(Clone)]
pub struct PyTableScan {
pub(crate) table_scan: TableScan,
input: Arc<LogicalPlan>,
}

#[pymethods]
Expand All @@ -31,14 +38,32 @@ impl PyTableScan {
fn contains_projections(&self) -> bool {
self.table_scan.projection.is_some()
}

#[pyo3(name = "getFilters")]
fn scan_filters(&self) -> PyResult<Vec<PyExpr>> {
py_expr_list(&self.input, &self.table_scan.filters)
}
}

impl TryFrom<logical::LogicalPlan> for PyTableScan {
impl TryFrom<LogicalPlan> for PyTableScan {
type Error = PyErr;

fn try_from(logical_plan: logical::LogicalPlan) -> Result<Self, Self::Error> {
fn try_from(logical_plan: LogicalPlan) -> Result<Self, Self::Error> {
match logical_plan {
logical::LogicalPlan::TableScan(table_scan) => Ok(PyTableScan { table_scan }),
LogicalPlan::TableScan(table_scan) => {
// Create an input logical plan that's identical to the table scan with schema from the table source
let mut input = table_scan.clone();
input.projected_schema = DFSchema::try_from_qualified_schema(
&table_scan.table_name,
&table_scan.source.schema(),
)
.map_or(input.projected_schema, Arc::new);

Ok(PyTableScan {
table_scan,
input: Arc::new(LogicalPlan::TableScan(input)),
})
}
_ => Err(py_type_err("unexpected plan")),
}
}
Expand Down
4 changes: 1 addition & 3 deletions dask_planner/src/sql/optimizer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use datafusion_optimizer::{
// eliminate_filter::EliminateFilter,
eliminate_limit::EliminateLimit,
filter_null_join_keys::FilterNullJoinKeys,
filter_push_down::FilterPushDown,
inline_table_scan::InlineTableScan,
limit_push_down::LimitPushDown,
optimizer::{Optimizer, OptimizerRule},
Expand All @@ -29,9 +30,6 @@ use log::trace;
mod eliminate_agg_distinct;
use eliminate_agg_distinct::EliminateAggDistinct;

mod filter_push_down;
use filter_push_down::FilterPushDown;

/// Houses the optimization logic for Dask-SQL. This optimization controls the optimizations
/// and their ordering in regards to their impact on the underlying `LogicalPlan` instance
pub struct DaskSqlOptimizer {
Expand Down
Loading