Allow setting ClientOptions with options API

object_store/src/aws/mod.rs

@@ -604,6 +604,9 @@ pub enum AmazonS3ConfigKey {
     /// - `aws_profile`
     /// - `profile`
     Profile,
+
+    /// Client options
+    Client(ClientConfigKey),
 }
 
 impl AsRef<str> for AmazonS3ConfigKey {
@@ -622,6 +625,7 @@ impl AsRef<str> for AmazonS3ConfigKey {
             Self::Profile => "aws_profile",
             Self::UnsignedPayload => "aws_unsigned_payload",
             Self::Checksum => "aws_checksum_algorithm",
+            Self::Client(opt) => opt.as_ref(),
         }
     }
 }
@@ -652,7 +656,12 @@ impl FromStr for AmazonS3ConfigKey {
             "aws_metadata_endpoint" | "metadata_endpoint" => Ok(Self::MetadataEndpoint),
             "aws_unsigned_payload" | "unsigned_payload" => Ok(Self::UnsignedPayload),
             "aws_checksum_algorithm" | "checksum_algorithm" => Ok(Self::Checksum),
-            _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+            // Backwards compatibility
+            "aws_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)),
+            _ => match s.parse() {
+                Ok(key) => Ok(Self::Client(key)),
+                Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+            },
         }
     }
 }
@@ -688,9 +697,7 @@ impl AmazonS3Builder {
         for (os_key, os_value) in std::env::vars_os() {
             if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
                 if key.starts_with("AWS_") {
-                    if let Ok(config_key) =
-                        AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase())
-                    {
+                    if let Ok(config_key) = key.to_ascii_lowercase().parse() {
                         builder = builder.with_config(config_key, value);
                     }
                 }
@@ -706,12 +713,6 @@ impl AmazonS3Builder {
                 Some(format!("{METADATA_ENDPOINT}{metadata_relative_uri}"));
         }
 
-        if let Ok(text) = std::env::var("AWS_ALLOW_HTTP") {
-            builder.client_options = builder
-                .client_options
-                .with_config(ClientConfigKey::AllowHttp, text);
-        }
-
         builder
     }
 
@@ -770,6 +771,9 @@ impl AmazonS3Builder {
             AmazonS3ConfigKey::Checksum => {
                 self.checksum_algorithm = Some(ConfigValue::Deferred(value.into()))
             }
+            AmazonS3ConfigKey::Client(key) => {
+                self.client_options = self.client_options.with_config(key, value)
+            }
         };
         self
     }
@@ -834,6 +838,7 @@ impl AmazonS3Builder {
             AmazonS3ConfigKey::Checksum => {
                 self.checksum_algorithm.as_ref().map(ToString::to_string)
             }
+            AmazonS3ConfigKey::Client(key) => self.client_options.get_config_value(key),
         }
     }
 

object_store/src/azure/mod.rs

@@ -559,6 +559,9 @@ pub enum AzureConfigKey {
     /// - `azure_use_azure_cli`
     /// - `use_azure_cli`
     UseAzureCli,
+
+    /// Client options
+    Client(ClientConfigKey),
 }
 
 impl AsRef<str> for AzureConfigKey {
@@ -577,6 +580,7 @@ impl AsRef<str> for AzureConfigKey {
             Self::MsiResourceId => "azure_msi_resource_id",
             Self::FederatedTokenFile => "azure_federated_token_file",
             Self::UseAzureCli => "azure_use_azure_cli",
+            Self::Client(key) => key.as_ref(),
         }
     }
 }
@@ -621,7 +625,12 @@ impl FromStr for AzureConfigKey {
                 Ok(Self::FederatedTokenFile)
             }
             "azure_use_azure_cli" | "use_azure_cli" => Ok(Self::UseAzureCli),
-            _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+            // Backwards compatibility
+            "azure_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)),
+            _ => match s.parse() {
+                Ok(key) => Ok(Self::Client(key)),
+                Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+            },
         }
     }
 }
@@ -673,12 +682,6 @@ impl MicrosoftAzureBuilder {
             }
         }
 
-        if let Ok(text) = std::env::var("AZURE_ALLOW_HTTP") {
-            builder.client_options = builder
-                .client_options
-                .with_config(ClientConfigKey::AllowHttp, text)
-        }
-
         if let Ok(text) = std::env::var(MSI_ENDPOINT_ENV_KEY) {
             builder = builder.with_msi_endpoint(text);
         }
@@ -731,6 +734,9 @@ impl MicrosoftAzureBuilder {
             }
             AzureConfigKey::UseAzureCli => self.use_azure_cli.parse(value),
             AzureConfigKey::UseEmulator => self.use_emulator.parse(value),
+            AzureConfigKey::Client(key) => {
+                self.client_options = self.client_options.with_config(key, value)
+            }
         };
         self
     }
@@ -786,6 +792,7 @@ impl MicrosoftAzureBuilder {
             AzureConfigKey::MsiResourceId => self.msi_resource_id.clone(),
             AzureConfigKey::FederatedTokenFile => self.federated_token_file.clone(),
             AzureConfigKey::UseAzureCli => Some(self.use_azure_cli.to_string()),
+            AzureConfigKey::Client(key) => self.client_options.get_config_value(key),
         }
     }
 

object_store/src/client/mod.rs

@@ -31,6 +31,7 @@ use reqwest::header::{HeaderMap, HeaderValue};
 use reqwest::{Client, ClientBuilder, Proxy};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::str::FromStr;
 use std::time::Duration;
 
 use crate::path::Path;
@@ -53,6 +54,28 @@ pub enum ClientConfigKey {
     AllowHttp,
 }
 
+impl AsRef<str> for ClientConfigKey {
+    fn as_ref(&self) -> &str {
+        match self {
+            ClientConfigKey::AllowHttp => "allow_http",
+        }
+    }
+}
+
+impl FromStr for ClientConfigKey {
+    type Err = super::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "allow_http" => Ok(Self::AllowHttp),
+            _ => Err(super::Error::UnknownConfigurationKey {
+                store: "HTTP",
+                key: s.into(),
+            }),
+        }
+    }
+}
+
 /// HTTP client configuration for remote object stores
 #[derive(Debug, Clone, Default)]
 pub struct ClientOptions {

object_store/src/gcp/mod.rs

@@ -49,6 +49,7 @@ use url::Url;
 
 use crate::client::pagination::stream_paginated;
 use crate::client::retry::RetryExt;
+use crate::client::ClientConfigKey;
 use crate::{
     client::token::TokenCache,
     multipart::{CloudMultiPartUpload, CloudMultiPartUploadImpl, UploadPart},
@@ -829,6 +830,9 @@ pub enum GoogleConfigKey {
     ///
     /// See [`GoogleCloudStorageBuilder::with_application_credentials`].
     ApplicationCredentials,
+
+    /// Client options
+    Client(ClientConfigKey),
 }
 
 impl AsRef<str> for GoogleConfigKey {
@@ -838,6 +842,7 @@ impl AsRef<str> for GoogleConfigKey {
             Self::ServiceAccountKey => "google_service_account_key",
             Self::Bucket => "google_bucket",
             Self::ApplicationCredentials => "google_application_credentials",
+            Self::Client(key) => key.as_ref(),
         }
     }
 }
@@ -858,7 +863,10 @@ impl FromStr for GoogleConfigKey {
                 Ok(Self::Bucket)
             }
             "google_application_credentials" => Ok(Self::ApplicationCredentials),
-            _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+            _ => match s.parse() {
+                Ok(key) => Ok(Self::Client(key)),
+                Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
+            },
         }
     }
 }
@@ -957,6 +965,9 @@ impl GoogleCloudStorageBuilder {
             GoogleConfigKey::ApplicationCredentials => {
                 self.application_credentials_path = Some(value.into())
             }
+            GoogleConfigKey::Client(key) => {
+                self.client_options = self.client_options.with_config(key, value)
+            }
         };
         self
     }
@@ -1005,6 +1016,7 @@ impl GoogleCloudStorageBuilder {
             GoogleConfigKey::ApplicationCredentials => {
                 self.application_credentials_path.clone()
             }
+            GoogleConfigKey::Client(key) => self.client_options.get_config_value(key),
         }
     }