Clean up code / deps

Cargo.toml

@@ -34,25 +34,17 @@ pyo3 = "0.22.0"
 reqwest = "0.12.3"
 env_logger = "0.11.3"
 thiserror = "1.0.58"
-base64 = "0.22.0"
 async-recursion = "1.1.0"
 async-trait = "0.1.81"
-crossbeam = "0.8.2"
 chrono = "0.4.37"
 filesize = "0.2.0"
-futures = "0.3.30"
-tokio-stream = "0.1.15"
 serde = "1.0.203"
 serde_json = "1.0.117"
 backoff = "0.4.0"
 gcp-bigquery-client = "0.20.0"
 rayon = "1.10.0"
 opcua = {version="0.12.0", features = ["vendored-openssl"]}
 url = "2.5.2"
-bollard = "0.15.0"
-rstest = "0.21.0"
-serial_test = "3.1.1"
-futures-util = "0.3.30"
 
 [patch.crates-io]
 oxrdf = { git = 'https://github.com/oxigraph/oxigraph.git', rev = "1e08089454a9aa8b3f66f4a422aed27e73faae84"}

lib/bigquery-polars/Cargo.toml

@@ -7,6 +7,5 @@ edition = "2021"
 gcp-bigquery-client = {workspace = true, features = ["rust-tls"]}
 polars = {workspace = true, features = ["dtype-full", "cse", "nightly", "performant", "timezones", "lazy"]}
 thiserror.workspace = true
-backoff.workspace = true
 tokio = {workspace = true, features = ["time"]}
 rayon.workspace = true

lib/chrontext/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2021"
 virtualization = {path = "../virtualization"}
 virtualized_query = { path = "../virtualized_query" }
 
-templates.workspace = true
 polars = { workspace = true, features = [
     "lazy",
     "concat_str",
@@ -21,31 +20,16 @@ polars = { workspace = true, features = [
     "cse",
     "nightly",
     "performant"] }
-tokio = { workspace = true, features = ["rt-multi-thread", "rt"] }
 log.workspace = true
 spargebra.workspace = true
 representation.workspace=true
 query_processing.workspace = true
 sparesults.workspace = true
 oxrdf.workspace = true
 reqwest = { workspace = true, features = ["stream"] }
-env_logger.workspace = true
 thiserror.workspace = true
 async-recursion.workspace = true
 async-trait.workspace = true
 oxigraph.workspace = true
-chrono = { workspace = true, features = ["clock"] }
 serde.workspace = true
-serde_json.workspace = true
 filesize.workspace = true
-pyo3.workspace = true
-
-[dev-dependencies]
-bollard.workspace = true
-rstest.workspace = true
-polars = { workspace = true, features = ["lazy", "unique_counts"] }
-tokio = { workspace = true, features = ["rt-multi-thread"] }
-serial_test.workspace = true
-futures-util.workspace = true
-reqwest = { workspace = true, features = ["stream", "json"] }
-opcua.workspace = true

lib/chrontext/src/combiner/virtualized_queries.rs

@@ -15,6 +15,7 @@ use representation::{BaseRDFNodeType, RDFNodeType};
 use sparesults::QuerySolution;
 use std::collections::{HashMap, HashSet};
 use virtualized_query::{BasicVirtualizedQuery, VirtualizedQuery};
+use virtualized_query::pushdown_setting::PushdownSetting;
 
 impl Combiner {
     pub fn attach_expected_empty_results(&self, vq:&VirtualizedQuery, mut solution_mappings: SolutionMappings) -> SolutionMappings {
@@ -61,9 +62,12 @@ impl Combiner {
             return Ok(self.attach_expected_empty_results(&vq, solution_mappings))
         }
 
-        let on_cols = get_join_columns(&vq, &solution_mappings.rdf_node_types);
         //Find the columns we should join on:
-        vq = vq.add_sorting_pushdown(&on_cols);
+        let on_cols = get_join_columns(&vq, &solution_mappings.rdf_node_types);
+
+        if self.prepper.pushdown_settings.contains(&PushdownSetting::Ordering) {
+            vq = vq.add_sorting_pushdown(&on_cols);
+        }
         let EagerSolutionMappings {
             mappings,
             mut rdf_node_types,

lib/chrontext/src/preparing.rs

@@ -15,7 +15,7 @@ use virtualized_query::{BasicVirtualizedQuery, VirtualizedQuery};
 
 #[derive(Debug)]
 pub struct TimeseriesQueryPrepper {
-    pushdown_settings: HashSet<PushdownSetting>,
+    pub(crate) pushdown_settings: HashSet<PushdownSetting>,
     pub(crate) basic_virtualized_queries: Vec<BasicVirtualizedQuery>,
     grouping_counter: u16,
     rewritten_filters: HashMap<Context, Expression>,

lib/virtualization/Cargo.toml

@@ -17,7 +17,6 @@ polars = {workspace=true, features=[
     "nightly",
     "performant"] }
 opcua.workspace= true
-log.workspace = true
 oxrdf.workspace = true
 query_processing.workspace = true
 spargebra.workspace=true

lib/virtualization/src/bigquery.rs

@@ -159,7 +159,7 @@ fn rename_non_alpha_vars(vq: VirtualizedQuery, rename_map: &mut HashMap<Variable
             rename_non_alpha_expr_vars(&mut e, rename_map);
             VirtualizedQuery::ExpressionAs(Box::new(new_vq), v, e)
         }
-        VirtualizedQuery::Grouped(GroupedVirtualizedQuery{ context, vq, by, mut aggregations }) => {
+        VirtualizedQuery::Grouped(GroupedVirtualizedQuery{ context, vq, by, aggregations }) => {
             let new_vq = rename_non_alpha_vars(*vq, rename_map);
             let mut new_by = vec![];
             for v in by {

lib/virtualization/src/opcua.rs

@@ -92,7 +92,7 @@ impl VirtualizedOPCUADatabase {
         let mut colnames_identifiers = vec![];
         let mut grouping_col_lookup = HashMap::new();
         let grouping_columns = vq.get_groupby_columns();
-        let mut grouping_col_name = if let Some(g) = grouping_columns.get(0) {
+        let grouping_col_name = if let Some(g) = grouping_columns.get(0) {
             Some(g.deref().clone())
         } else {
             None

lib/virtualization/src/python.rs

@@ -1,3 +1,5 @@
+mod sql_translation;
+
 use polars::prelude::DataFrame;
 use pydf_io::to_rust::polars_df_to_rust_df;
 use pyo3::prelude::*;
@@ -6,6 +8,7 @@ use std::collections::{HashSet};
 use virtualized_query::pushdown_setting::{all_pushdowns, PushdownSetting};
 use virtualized_query::python::PyVirtualizedQuery;
 use virtualized_query::VirtualizedQuery;
+use sql_translation::PYTHON_CODE;
 
 #[derive(Clone, Debug)]
 #[pyclass]
@@ -55,7 +58,7 @@ impl VirtualizedPythonDatabase {
 pub fn translate_sql(vq: &VirtualizedQuery, resource_sql_map: &Py<PyDict>, dialect:&str) -> PyResult<String> {
     Python::with_gil(|py| {
         let pyvq = PyVirtualizedQuery::new(vq.clone(), py)?;
-        let db_mod = PyModule::import_bound(py, "my_db")?;
+        let db_mod = PyModule::from_code_bound(py, PYTHON_CODE, "my_translator", "my_translator")?;
         let translate_sql_func = db_mod.getattr("translate_sql")?;
         let query_string = translate_sql_func.call((pyvq, dialect, resource_sql_map), None)?;
         query_string.extract::<String>()