From 084345ebd501cd772e642640a673f64b8524a8af Mon Sep 17 00:00:00 2001 From: zhangli20 Date: Tue, 17 Sep 2024 21:01:01 +0800 Subject: [PATCH] blaze: make parquet reader components public --- .../core/src/datasource/physical_plan/file_scan_config.rs | 2 +- datafusion/core/src/datasource/physical_plan/parquet/mod.rs | 4 ++-- .../core/src/datasource/physical_plan/parquet/opener.rs | 2 +- datafusion/core/src/datasource/schema_adapter.rs | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datafusion/core/src/datasource/physical_plan/file_scan_config.rs b/datafusion/core/src/datasource/physical_plan/file_scan_config.rs index 27f388778be1..5b279d19bba5 100644 --- a/datafusion/core/src/datasource/physical_plan/file_scan_config.rs +++ b/datafusion/core/src/datasource/physical_plan/file_scan_config.rs @@ -288,7 +288,7 @@ impl FileScanConfig { ) } - pub(crate) fn file_column_projection_indices(&self) -> Option> { + pub fn file_column_projection_indices(&self) -> Option> { self.projection.as_ref().map(|p| { p.iter() .filter(|col_idx| **col_idx < self.file_schema.fields().len()) diff --git a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs index 54d4d7262a8e..f26d54761c78 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs @@ -48,7 +48,7 @@ use log::debug; mod access_plan; mod metrics; mod opener; -mod page_filter; +pub mod page_filter; mod reader; mod row_filter; mod row_group_filter; @@ -59,7 +59,7 @@ use crate::datasource::schema_adapter::{ }; pub use access_plan::{ParquetAccessPlan, RowGroupAccess}; pub use metrics::ParquetFileMetrics; -use opener::ParquetOpener; +pub use opener::ParquetOpener; pub use reader::{DefaultParquetFileReaderFactory, ParquetFileReaderFactory}; pub use writer::plan_to_parquet; diff --git a/datafusion/core/src/datasource/physical_plan/parquet/opener.rs b/datafusion/core/src/datasource/physical_plan/parquet/opener.rs index 3766be9f7ec2..4e207ef9183a 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/opener.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/opener.rs @@ -40,7 +40,7 @@ use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask}; use std::sync::Arc; /// Implements [`FileOpener`] for a parquet file -pub(super) struct ParquetOpener { +pub struct ParquetOpener { pub partition_index: usize, pub projection: Arc<[usize]>, pub batch_size: usize, diff --git a/datafusion/core/src/datasource/schema_adapter.rs b/datafusion/core/src/datasource/schema_adapter.rs index a293227237d1..f887d1a906fb 100644 --- a/datafusion/core/src/datasource/schema_adapter.rs +++ b/datafusion/core/src/datasource/schema_adapter.rs @@ -173,9 +173,9 @@ impl SchemaAdapter for DefaultSchemaAdapter { #[derive(Debug)] pub struct SchemaMapping { /// The schema of the table. This is the expected schema after conversion and it should match the schema of the query result. - table_schema: SchemaRef, + pub table_schema: SchemaRef, /// Mapping from field index in `table_schema` to index in projected file_schema - field_mappings: Vec>, + pub field_mappings: Vec>, } impl SchemaMapper for SchemaMapping {