Add a feature for hdfs3 (apache#439)

* Add a feature for hdfs3 * Update version for dependency of datafusion-objectstore-hdfs Co-authored-by: yangzhong <[email protected]>
sxt-partners · Oct 24, 2022 · eec5c18 · eec5c18
1 parent 8c76051
commit eec5c18
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 4 deletions.
diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml
@@ -43,5 +43,6 @@ tokio = "1.0"
 [features]
 default = []
 hdfs = ["ballista-core/hdfs"]
+hdfs3 = ["ballista-core/hdfs3"]
 s3 = ["ballista-core/s3"]
 standalone = ["ballista-executor", "ballista-scheduler"]
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
@@ -34,7 +34,8 @@ rustc-args = ["--cfg", "docsrs"]
 # Used for testing ONLY: causes all values to hash to the same value (test for collisions)
 force_hash_collisions = ["datafusion/force_hash_collisions"]
 # Used to enable hdfs to be registered in the ObjectStoreRegistry by default
-hdfs = ["datafusion-objectstore-hdfs"]
+hdfs = ["datafusion-objectstore-hdfs/hdfs"]
+hdfs3 = ["datafusion-objectstore-hdfs/hdfs3"]
 s3 = ["object_store/aws"]
 simd = ["datafusion/simd"]
 
@@ -46,7 +47,7 @@ async-trait = "0.1.41"
 chrono = { version = "0.4", default-features = false }
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = "13.0.0"
-datafusion-objectstore-hdfs = { version = "0.1.0", optional = true }
+datafusion-objectstore-hdfs = { version = "0.1.1", default-features = false, optional = true }
 datafusion-proto = "13.0.0"
 futures = "0.3"
 hashbrown = "0.12"

diff --git a/ballista/core/src/utils.rs b/ballista/core/src/utils.rs
@@ -43,7 +43,7 @@ use datafusion::physical_plan::metrics::MetricsSet;
 use datafusion::physical_plan::projection::ProjectionExec;
 use datafusion::physical_plan::sorts::sort::SortExec;
 use datafusion::physical_plan::{metrics, ExecutionPlan, RecordBatchStream};
-#[cfg(feature = "hdfs")]
+#[cfg(any(feature = "hdfs", feature = "hdfs3"))]
 use datafusion_objectstore_hdfs::object_store::hdfs::HadoopFileSystem;
 use datafusion_proto::logical_plan::{
     AsLogicalPlan, DefaultLogicalExtensionCodec, LogicalExtensionCodec,
@@ -89,7 +89,7 @@ impl ObjectStoreProvider for FeatureBasedObjectStoreProvider {
     /// Return the key and object store
     #[allow(unused_variables)]
     fn get_by_url(&self, url: &Url) -> datafusion::error::Result<Arc<dyn ObjectStore>> {
-        #[cfg(feature = "hdfs")]
+        #[cfg(any(feature = "hdfs", feature = "hdfs3"))]
         {
             let store = HadoopFileSystem::new(url.as_str());
             if let Some(store) = store {