Skip to content

Commit

Permalink
fix: new exon to fix overflow (#135)
Browse files Browse the repository at this point in the history
  • Loading branch information
tshauck authored May 21, 2024
1 parent 7926177 commit c4ad2eb
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 4 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ name = "biobear"
[dependencies]
arrow = { version = "51.0.0", features = ["pyarrow"] }
datafusion = "38"
exon = { version = "0.21", features = ["default"] }
exon = { version = "0.21.1", features = ["default"] }
pyo3 = "0.20"
tokio = { version = "1", features = ["rt"] }
noodles = { version = "0.73", features = ["core"] }
Expand Down
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ build:
cargo build
maturin develop

release-build:
cargo build --release
maturin develop --release

run-benchmarks:
hyperfine --runs 2 \
-n biopython 'python benchmarks/biopython-scan.py' \
Expand Down
17 changes: 14 additions & 3 deletions src/execution_result.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,21 @@ impl ExecutionResult {

/// Convert to a Polars DataFrame
fn to_polars(&self, py: Python) -> PyResult<PyObject> {
let batches = self.collect(py)?.to_object(py);
let schema = self.schema().into_py(py);
let stream = wait_for_future(py, self.df.as_ref().clone().execute_stream())
.map_err(error::BioBearError::from)?;

let runtime = Arc::new(Runtime::new()?);

let dataframe_record_batch_stream = DataFrameRecordBatchStream::new(stream, runtime);

let mut stream = FFI_ArrowArrayStream::new(Box::new(dataframe_record_batch_stream));

let schema = schema.into_py(py);
let batches =
unsafe { ArrowArrayStreamReader::from_raw(&mut stream).map_err(BioBearError::from) }?;

let batches = batches.into_pyarrow(py)?;

let schema = self.schema().into_py(py);

let table_class = py.import("pyarrow")?.getattr("Table")?;
let args = (batches, schema);
Expand Down

0 comments on commit c4ad2eb

Please sign in to comment.