Merge pull request #1036 from TileDB-Inc/ss/1030-add-aws-key-configs

Support AWS credentials in tiledb config

HISTORY.md

@@ -4,6 +4,7 @@
 
 ## Improvements
 
+* Added config params `vfs.s3.aws_access_key_id` and `vfs.s3.aws_secret_access_key` for configure s3 access at runtime. [#1036](https://github.com/TileDB-Inc/TileDB/pull/1036)
 * Set LZ4, Zlib and Zstd compressors to build in release mode. [#1034](https://github.com/TileDB-Inc/TileDB/pull/1034)
 
 # TileDB v1.4.0 Release Notes

test/src/unit-capi-config.cc

@@ -175,6 +175,16 @@ void check_save_to_file() {
   REQUIRE(rc == TILEDB_OK);
   CHECK(error == nullptr);
 
+  // Check that aws access key id is not serialized.
+  rc = tiledb_config_set(config, "vfs.s3.aws_access_key_id", "keyid", &error);
+  REQUIRE(rc == TILEDB_OK);
+  CHECK(error == nullptr);
+  // Check that aws secret access key is not serialized.
+  rc = tiledb_config_set(
+      config, "vfs.s3.aws_secret_access_key", "secret", &error);
+  REQUIRE(rc == TILEDB_OK);
+  CHECK(error == nullptr);
+
   rc = tiledb_config_save_to_file(config, "test_config.txt", &error);
   REQUIRE(rc == TILEDB_OK);
 
@@ -368,6 +378,8 @@ TEST_CASE("C API: Test config iter", "[capi], [config]") {
       std::to_string(std::thread::hardware_concurrency());
   all_param_values["vfs.s3.scheme"] = "https";
   all_param_values["vfs.s3.region"] = "us-east-1";
+  all_param_values["vfs.s3.aws_access_key_id"] = "";
+  all_param_values["vfs.s3.aws_secret_access_key"] = "";
   all_param_values["vfs.s3.endpoint_override"] = "";
   all_param_values["vfs.s3.use_virtual_addressing"] = "true";
   all_param_values["vfs.s3.max_parallel_ops"] =
@@ -394,6 +406,8 @@ TEST_CASE("C API: Test config iter", "[capi], [config]") {
       std::to_string(std::thread::hardware_concurrency());
   vfs_param_values["s3.scheme"] = "https";
   vfs_param_values["s3.region"] = "us-east-1";
+  vfs_param_values["s3.aws_access_key_id"] = "";
+  vfs_param_values["s3.aws_secret_access_key"] = "";
   vfs_param_values["s3.endpoint_override"] = "";
   vfs_param_values["s3.use_virtual_addressing"] = "true";
   vfs_param_values["s3.max_parallel_ops"] =
@@ -415,6 +429,8 @@ TEST_CASE("C API: Test config iter", "[capi], [config]") {
   std::map<std::string, std::string> s3_param_values;
   s3_param_values["scheme"] = "https";
   s3_param_values["region"] = "us-east-1";
+  s3_param_values["aws_access_key_id"] = "";
+  s3_param_values["aws_secret_access_key"] = "";
   s3_param_values["endpoint_override"] = "";
   s3_param_values["use_virtual_addressing"] = "true";
   s3_param_values["max_parallel_ops"] =

test/src/unit-cppapi-config.cc

@@ -50,5 +50,5 @@ TEST_CASE("C++ API: Config iterator", "[cppapi], [cppapi-config]") {
     names.push_back(it->first);
   }
   // Check number of VFS params in default config object.
-  CHECK(names.size() == 21);
+  CHECK(names.size() == 23);
 }

tiledb/sm/filesystem/s3.cc

@@ -137,11 +137,23 @@ Status S3::init(const Config::S3Params& s3_config, ThreadPool* thread_pool) {
       s3_config.connect_scale_factor_);
 
   // Connect S3 client
-  client_ = Aws::MakeShared<Aws::S3::S3Client>(
-      constants::s3_allocation_tag.c_str(),
-      config,
-      Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-      s3_config.use_virtual_addressing_);
+  // If the user set config variables for aws keys use them
+  if (!s3_config.aws_access_key_id.empty() &&
+      !s3_config.aws_secret_access_key.empty()) {
+    client_ = Aws::MakeShared<Aws::S3::S3Client>(
+        constants::s3_allocation_tag.c_str(),
+        Aws::Auth::AWSCredentials(
+            s3_config.aws_access_key_id, s3_config.aws_secret_access_key),
+        config,
+        Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+        s3_config.use_virtual_addressing_);
+  } else {
+    client_ = Aws::MakeShared<Aws::S3::S3Client>(
+        constants::s3_allocation_tag.c_str(),
+        config,
+        Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+        s3_config.use_virtual_addressing_);
+  }
 
   return Status::Ok();
 }

tiledb/sm/misc/constants.cc

@@ -446,6 +446,12 @@ const uint64_t s3_multipart_part_size = 5 * 1024 * 1024;
 /** S3 region. */
 const std::string s3_region = "us-east-1";
 
+/** S3 aws access key id. */
+const std::string aws_access_key_id = "";
+
+/** S3 aws secret access key. */
+const std::string aws_secret_access_key = "";
+
 /** S3 endpoint override. */
 const std::string s3_endpoint_override = "";
 

tiledb/sm/misc/constants.h

@@ -427,6 +427,12 @@ extern const uint64_t s3_multipart_part_size;
 /** S3 region. */
 extern const std::string s3_region;
 
+/** S3 aws access key id. */
+extern const std::string aws_access_key_id;
+
+/** S3 aws secret access key. */
+extern const std::string aws_secret_access_key;
+
 /** S3 endpoint override. */
 extern const std::string s3_endpoint_override;
 

tiledb/sm/storage_manager/config.cc

@@ -49,7 +49,10 @@ namespace sm {
 const char Config::COMMENT_START = '#';
 
 const std::set<std::string> Config::unserialized_params_ = {
-    "vfs.s3.proxy_username", "vfs.s3.proxy_password"};
+    "vfs.s3.proxy_username",
+    "vfs.s3.proxy_password",
+    "vfs.s3.aws_access_key_id",
+    "vfs.s3.aws_secret_access_key"};
 
 /* ****************************** */
 /*   CONSTRUCTORS & DESTRUCTORS   */
@@ -183,6 +186,10 @@ Status Config::set(const std::string& param, const std::string& value) {
     RETURN_NOT_OK(set_vfs_file_max_parallel_ops(value));
   } else if (param == "vfs.s3.region") {
     RETURN_NOT_OK(set_vfs_s3_region(value));
+  } else if (param == "vfs.s3.aws_access_key_id") {
+    RETURN_NOT_OK(set_vfs_s3_aws_access_key_id(value));
+  } else if (param == "vfs.s3.aws_secret_access_key") {
+    RETURN_NOT_OK(set_vfs_s3_aws_secret_access_key(value));
   } else if (param == "vfs.s3.scheme") {
     RETURN_NOT_OK(set_vfs_s3_scheme(value));
   } else if (param == "vfs.s3.endpoint_override") {
@@ -325,6 +332,16 @@ Status Config::unset(const std::string& param) {
     value << vfs_params_.s3_params_.region_;
     param_values_["vfs.s3.region"] = value.str();
     value.str(std::string());
+  } else if (param == "vfs.s3.aws_access_key_id") {
+    vfs_params_.s3_params_.aws_access_key_id = "";
+    value << vfs_params_.s3_params_.aws_access_key_id;
+    param_values_["vfs.s3.aws_access_key_id"] = value.str();
+    value.str(std::string());
+  } else if (param == "vfs.s3.aws_secret_access_key") {
+    vfs_params_.s3_params_.aws_secret_access_key = "";
+    value << vfs_params_.s3_params_.aws_secret_access_key;
+    param_values_["vfs.s3.aws_secret_access_key"] = value.str();
+    value.str(std::string());
   } else if (param == "vfs.s3.scheme") {
     vfs_params_.s3_params_.scheme_ = constants::s3_scheme;
     value << vfs_params_.s3_params_.scheme_;
@@ -490,6 +507,14 @@ void Config::set_default_param_values() {
   param_values_["vfs.s3.region"] = value.str();
   value.str(std::string());
 
+  value << vfs_params_.s3_params_.aws_access_key_id;
+  param_values_["vfs.s3.aws_access_key_id"] = value.str();
+  value.str(std::string());
+
+  value << vfs_params_.s3_params_.aws_secret_access_key;
+  param_values_["vfs.s3.aws_secret_access_key"] = value.str();
+  value.str(std::string());
+
   value << vfs_params_.s3_params_.scheme_;
   param_values_["vfs.s3.scheme"] = value.str();
   value.str(std::string());
@@ -696,6 +721,16 @@ Status Config::set_vfs_s3_region(const std::string& value) {
   return Status::Ok();
 }
 
+Status Config::set_vfs_s3_aws_access_key_id(const std::string& value) {
+  vfs_params_.s3_params_.aws_access_key_id = value;
+  return Status::Ok();
+}
+
+Status Config::set_vfs_s3_aws_secret_access_key(const std::string& value) {
+  vfs_params_.s3_params_.aws_secret_access_key = value;
+  return Status::Ok();
+}
+
 Status Config::set_vfs_s3_scheme(const std::string& value) {
   if (value != "http" && value != "https")
     return LOG_STATUS(

tiledb/sm/storage_manager/config.h

@@ -95,6 +95,8 @@ class Config {
     unsigned proxy_port_;
     std::string proxy_username_;
     std::string proxy_password_;
+    std::string aws_access_key_id;
+    std::string aws_secret_access_key;
 
     S3Params() {
       region_ = constants::s3_region;
@@ -112,6 +114,8 @@ class Config {
       proxy_port_ = constants::s3_proxy_port;
       proxy_username_ = constants::s3_proxy_username;
       proxy_password_ = constants::s3_proxy_password;
+      aws_access_key_id = constants::aws_access_key_id;
+      aws_secret_access_key = constants::aws_secret_access_key;
     }
   };
 
@@ -245,6 +249,10 @@ class Config {
    * - `vfs.s3.region` <br>
    *    The S3 region, if S3 is enabled. <br>
    *    **Default**: us-east-1
+   * - `vfs.s3.aws_access_key_id` <br>
+   *    Set the AWS_ACCESS_KEY_ID
+   * - `vfs.s3.aws_secret_access_key` <br>
+   *    Set the AWS_SECRET_ACCESS_KEY
    * - `vfs.s3.scheme` <br>
    *    The S3 scheme (`http` or `https`), if S3 is enabled. <br>
    *    **Default**: https
@@ -403,6 +411,12 @@ class Config {
   /** Sets the S3 region. */
   Status set_vfs_s3_region(const std::string& value);
 
+  /** Sets the S3 AWS_ACCESS_KEY_ID. */
+  Status set_vfs_s3_aws_access_key_id(const std::string& value);
+
+  /** Sets the S3 AWS_SECRET_ACCESS_KEY. */
+  Status set_vfs_s3_aws_secret_access_key(const std::string& value);
+
   /** Sets the S3 scheme. */
   Status set_vfs_s3_scheme(const std::string& value);