Skip to content

Commit

Permalink
Fix PR review
Browse files Browse the repository at this point in the history
  • Loading branch information
kyotoYaho committed Jul 17, 2022
1 parent 89841a9 commit bb9044b
Showing 1 changed file with 6 additions and 14 deletions.
20 changes: 6 additions & 14 deletions datafusion/core/src/datasource/object_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ impl std::fmt::Display for ObjectStoreUrl {
}
}

/// Object store self detector can detector an object store based on the url
pub trait ObjectStoreSelfDetector: Send + Sync + 'static {
/// Object store provider can detector an object store based on the url
pub trait ObjectStoreProvider: Send + Sync + 'static {
/// Detector a suitable object store based on its url if possible
/// Return the key and object store
fn get_by_url(&self, url: &Url) -> Option<(String, Arc<dyn ObjectStore>)>;
Expand All @@ -93,7 +93,7 @@ pub trait ObjectStoreSelfDetector: Send + Sync + 'static {
pub struct ObjectStoreRegistry {
/// A map from scheme to object store that serve list / read operations for the store
object_stores: Arc<RwLock<HashMap<String, Arc<dyn ObjectStore>>>>,
self_detector: Option<Arc<dyn ObjectStoreSelfDetector>>,
self_detector: Option<Arc<dyn ObjectStoreProvider>>,
}

impl std::fmt::Debug for ObjectStoreRegistry {
Expand Down Expand Up @@ -122,7 +122,7 @@ impl ObjectStoreRegistry {
/// Create the registry that object stores can registered into.
/// ['LocalFileSystem'] store is registered in by default to support read local files natively.
pub fn new_with_detector(
self_detector: Option<Arc<dyn ObjectStoreSelfDetector>>,
self_detector: Option<Arc<dyn ObjectStoreProvider>>,
) -> Self {
let mut map: HashMap<String, Arc<dyn ObjectStore>> = HashMap::new();
map.insert("file://".to_string(), Arc::new(LocalFileSystem::new()));
Expand All @@ -149,24 +149,16 @@ impl ObjectStoreRegistry {
///
/// - URL with scheme `file:///` or no schema will return the default LocalFS store
/// - URL with scheme `s3://bucket/` will return the S3 store if it's registered
/// - URL with scheme `hdfs://hostname:port` will return the hdfs store if it's registered
/// - URL with scheme `hdfs://hostname:port/` will return the hdfs store if it's registered
///
pub fn get_by_url(&self, url: impl AsRef<Url>) -> Result<Arc<dyn ObjectStore>> {
let url = url.as_ref();
// First check whether can get object store from registry
let store = {
let stores = self.object_stores.read();
let s = &url[url::Position::BeforeScheme..url::Position::BeforeHost];
let s = &url[url::Position::BeforeScheme..url::Position::BeforePath];
stores
.get(s)
.or_else(|| {
let s = &url[url::Position::BeforeScheme..url::Position::AfterHost];
stores.get(s)
})
.or_else(|| {
let s = &url[url::Position::BeforeScheme..url::Position::AfterPort];
stores.get(s)
})
.cloned()
};

Expand Down

0 comments on commit bb9044b

Please sign in to comment.