switch vendor from aws-sdk-js to botocore (#852)

* migrate vendor api from aws-sdk-js to botocore

* add botocore to submodules

* migrate vendor from aws-sdk-js to botocore

* update services and remove deprecated aws services

* remove aws-sdk-js as vendor api

* update r docs to reflect botocore

* fixed usage

* typo

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "vendor/aws-sdk-js"]
-	path = vendor/aws-sdk-js
-	url = https://github.com/aws/aws-sdk-js
+[submodule "vendor/botocore"]
+	path = vendor/botocore
+	url = https://github.com/boto/botocore

Makefile

@@ -1,4 +1,4 @@
-IN_DIR := ./vendor/aws-sdk-js
+IN_DIR := ./vendor/botocore
 OUT_DIR := ./cran
 DOC_DIR := ./paws
 CRAN_DIR := ./cran

make.paws/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: make.paws
 Type: Package
 Title: Generate Paws AWS SDKs for R
-Version: 0.9.2
+Version: 0.10.0
 Authors@R: c(
     person("David", "Kretch", email = "[email protected]", role = "aut"),
     person("Adam", "Banker", email = "[email protected]", role = "aut"),

make.paws/R/aws_services.R

@@ -3,13 +3,13 @@ NULL
 
 #' @import data.table
 
-#' @title Helper function to list all aws services and operations from awds js sdk
+#' @title Helper function to list all aws services and operations from aws botocore
 #'
-#' @param in_dir Directory containing aws js sdk.
+#' @param in_dir Directory containing aws botocore.
 #'
 #' @export
-list_aws_services <- function(in_dir = "../vendor/aws-sdk-js") {
-  apis <- list_apis(file.path(in_dir, "apis"))
+list_aws_services <- function(in_dir = "../vendor/botocore") {
+  apis <- list_apis(fs::path(in_dir, "botocore", "data"))
   aws_service_ops <- lapply(apis, function(x) {
     api <- read_api(x, in_dir)
     data.frame(
@@ -24,10 +24,10 @@ list_aws_services <- function(in_dir = "../vendor/aws-sdk-js") {
 
 #' @title Helper function to count all operations by service and category
 #'
-#' @param in_dir Directory containing aws js sdk.
+#' @param in_dir Directory containing aws botocore.
 #'
 #' @export
-category_service_ops_count <- function(in_dir = "../vendor/aws-sdk-js") {
+category_service_ops_count <- function(in_dir = "../vendor/botocore") {
   aws_service_ops <- list_aws_services(in_dir)
   paws_cat <- lapply(get_categories(), function(x) {
     data.frame(name = rep(x$name, length(x$services)), services = x$services)

make.paws/R/make_sdk.R

@@ -12,14 +12,14 @@
 #' @name make_sdk
 #' @export
 make_sdk <- function(
-    in_dir = "./vendor/aws-sdk-js",
+    in_dir = "./vendor/botocore",
     out_sdk_dir = "./cran",
     out_doc_dir = "./paws",
     apis = character(0),
     only_cran = TRUE,
     cache_dir = "./cache") {
   if (length(apis) == 0) {
-    apis <- list_apis(file.path(in_dir, "apis"))
+    apis <- list_apis(fs::path(in_dir, "botocore", "data"))
   }
 
   # Get the category-level packages that the SDK is separated into.
@@ -97,12 +97,17 @@ write_skeleton <- function(path, version) {
 }
 
 # Return the names of the APIs available in the given directory.
-list_apis <- function(path) {
+list_apis_old <- function(path) {
   files <- list.files(file.path(path), pattern = ".*\\.normal\\.json")
   apis <- unique(gsub("(.+)-\\d{4}.*", "\\1", files))
   return(apis)
 }
 
+list_apis <- function(path) {
+  return(fs::path_file(fs::dir_ls(path, type = "dir")))
+}
+
+
 # Create the package's R directory.
 use_r_dir <- function(path) {
   r_dir <- file.path(path, "R")
@@ -145,15 +150,15 @@ use_package_doc <- function(path) {
   package <- methods::getPackageName()
   template <- system_file("templates/package.R", package = package)
   to <- file.path(path, "R", paste0(basename(path), "_package.R"))
-  fs::file_copy(template, to, overwrite = T)
+  fs::file_copy(template, to, overwrite = TRUE)
 }
 
 # Create a dummy NAMESPACE file.
 use_namespace <- function(path) {
   package <- methods::getPackageName()
   template <- system_file("templates/NAMESPACE", package = package)
   to <- file.path(path, "NAMESPACE")
-  fs::file_copy(template, to, overwrite = T)
+  fs::file_copy(template, to, overwrite = TRUE)
 }
 
 # Return the authors from this package.

make.paws/R/paws_helper_fun.R

@@ -65,10 +65,10 @@ paws_service_param_fn <- function(dir = "../paws.common") {
 #' @title lists new services from AWS
 #' @param in_dir Directory containing AWS API input files.
 #' @export
-paws_new_services <- function(in_dir = "../vendor/aws-sdk-js") {
+paws_new_services <- function(in_dir = "../vendor/botocore") {
   existing_apis <- yaml::read_yaml(system.file("extdata/packages.yml", package = "make.paws"))
   existing_apis <- unlist(lapply(existing_apis, \(x) x$services))
 
-  new_apis <- list_apis(file.path(in_dir, "apis"))
+  new_apis <- list_apis(file.path(in_dir, "botocore", "data"))
   return(setdiff(new_apis, existing_apis))
 }

make.paws/R/read_api.R

@@ -1,7 +1,7 @@
 # Read a given API's definition and documentation files.
 # aws-sdk-js deprecated and apis is not being updated
 # TODO: short term migrate to botocore jsons
-read_api <- function(api_name, path) {
+read_api_old <- function(api_name, path) {
   api_path <- file.path(path, "apis")
   region_config_path <- file.path(path, "lib/region_config_data.json")
 
@@ -28,6 +28,33 @@ read_api <- function(api_name, path) {
   return(api)
 }
 
+read_api <- function(api_name, path) {
+  api_path <- file.path(path, "botocore", "data")
+  region_config_path <- file.path(api_path, "endpoints.json")
+
+  version <- get_latest_api_version_v2(api_name, api_path)
+  files <- get_api_files_v2(version, api_name, api_path)
+  if (length(version) == 0 || length(files) == 0) stop("Invalid API")
+
+  api <- jsonlite::read_json(files$service)
+  api <- fix_operation_names(api)
+
+  if (!is.null(files$examples)) {
+    examples <- jsonlite::read_json(files$examples)
+    api <- merge_examples(api, examples$examples)
+  }
+  if (!is.null(files$paginators)) {
+    paginators <- jsonlite::read_json(files$paginators)
+    api <- merge_paginators(api, paginators$pagination)
+  }
+  api <- merge_eventstream(api)
+  region_config <- jsonlite::read_json(region_config_path)
+  api <- merge_region_config_v2(api, region_config)
+  api <- fix_region_config(api)
+
+  return(api)
+}
+
 # Returns the latest version of the given API.
 get_latest_api_version <- function(name, path) {
   files <- list.files(path, pattern = sprintf("^%s-.{10}.normal.json", name))
@@ -36,6 +63,12 @@ get_latest_api_version <- function(name, path) {
   return(latest)
 }
 
+get_latest_api_version_v2 <- function(api_name, path) {
+  dir_ls <- list.files(file.path(path, api_name))
+  if (length(dir_ls) == 0) stop("Invalid API")
+  return(max(as.Date(dir_ls)))
+}
+
 # Returns a list of API files for a given API version.
 get_api_files <- function(version, path) {
   files <- list.files(path, pattern = sprintf("^%s", version), full.names = TRUE)
@@ -45,6 +78,13 @@ get_api_files <- function(version, path) {
   return(files)
 }
 
+get_api_files_v2 <- function(version, api_name, api_path) {
+  files <- list.files(file.path(api_path, api_name, version), full.names = TRUE)
+  types <- gsub("(-\\d.json)$", "", basename(files))
+  names(files) <- types
+  return(as.list(files))
+}
+
 # Returns an API object with examples merged into the corresponding operations.
 merge_examples <- function(api, examples) {
   for (name in names(examples)) {
@@ -67,18 +107,18 @@ merge_paginators <- function(api, paginators) {
 
 merge_eventstream <- function(api) {
   flat_shape <- unlist(api$shapes)
-  eventstream <- flat_shape[endsWith(names(flat_shape),"eventstream")]
+  eventstream <- flat_shape[endsWith(names(flat_shape), "eventstream")]
   names(eventstream) <- stringr::str_extract(names(eventstream), "([a-zA-Z]+)")
 
   shape <- flat_shape[endsWith(names(flat_shape), "shape")]
   shape <- shape[shape %in% names(eventstream)]
   names(shape) <- gsub(
-    "Output$|Response$", "", stringr::str_extract(names(shape), "([a-zA-Z]+)")
+    "Output$|Response$|Request$", "", stringr::str_extract(names(shape), "([a-zA-Z]+)")
   )
   names(eventstream) <- names(shape)
 
-  for (nms in names(eventstream)) {
-    api$operations[[nms]]$eventstream <- eventstream[nms]
+  for (nms in unique(names(eventstream))) {
+    api$operations[[nms]]$eventstream <- eventstream[[nms]]
   }
   return(api)
 }
@@ -111,6 +151,34 @@ merge_region_config <- function(api, region_config) {
   return(api)
 }
 
+merge_region_config_v2 <- function(api, region_config) {
+  service <- service_name(api)
+  endpoint_prefix <- api$metadata$endpointPrefix
+  ep <- list()
+  for (partition in region_config$partitions) {
+    dnsSuffix <- partition$dnsSuffix
+    hostname <- partition$defaults$hostname
+    region_regex <- gsub("\\", "\\\\", partition$regionRegex, fixed = T)
+    if (!is.null(global <- partition$services[[endpoint_prefix]]$endpoints[["aws-global"]])) {
+      endpoint <- build_endpoint(endpoint_prefix, global$hostname, dnsSuffix)
+      endpoint <- list(endpoint = endpoint, global = TRUE)
+      ep[["aws-global"]] <- endpoint
+      if (!is.null(region_name <- global$credentialScope$region)) {
+        ep[[region_name]] <- endpoint
+      }
+    }
+    endpoint <- list(endpoint = build_endpoint(endpoint_prefix, hostname, dnsSuffix), global = FALSE)
+    ep[[region_regex]] <- endpoint
+  }
+  api$region_config <- ep
+  return(api)
+}
+
+build_endpoint <- function(service, hostname, dnsSuffix) {
+  endpoint <- gsub("{service}", service, hostname, fixed = TRUE)
+  gsub("{dnsSuffix}", dnsSuffix, endpoint, fixed = TRUE)
+}
+
 # Make sure each operation has an exportable name. CloudFront's operation `name`
 # element is incorrect in this regard, e.g. they have values like
 # "OperationName2017_01_01".