Merge pull request #1702 from api3dao/fix-secrets-parsing

Fix Terraform secrets parsing & URLs propagations
api3dao · Mar 29, 2023 · c006034 · c006034
2 parents 62f105f + 87e9f4d
commit c006034
Show file tree

Hide file tree

Showing 12 changed files with 67 additions and 23 deletions.
diff --git a/.changeset/fifty-otters-heal.md b/.changeset/fifty-otters-heal.md
@@ -0,0 +1,5 @@
+---
+'@api3/airnode-deployer': patch
+---
+
+Fix propagating of gateway URLs to heartbeat
diff --git a/.changeset/happy-cycles-press.md b/.changeset/happy-cycles-press.md
@@ -0,0 +1,5 @@
+---
+'@api3/airnode-deployer': patch
+---
+
+Fix parsing of secrets.env with comments
diff --git a/packages/airnode-deployer/terraform/aws/main.tf b/packages/airnode-deployer/terraform/aws/main.tf
@@ -67,7 +67,7 @@ module "startCoordinatorHttpGw" {
   secrets_file       = var.secrets_file
 
   environment_variables = {
-    HTTP_GATEWAY_URL           = module.httpGw[0].api_url
+    HTTP_GATEWAY_URL           = local.http_gateway_url
     AIRNODE_WALLET_PRIVATE_KEY = var.airnode_wallet_private_key
   }
 
@@ -91,7 +91,7 @@ module "startCoordinatorHttpSignedGw" {
   secrets_file       = var.secrets_file
 
   environment_variables = {
-    HTTP_SIGNED_DATA_GATEWAY_URL = module.httpSignedGw[0].api_url
+    HTTP_SIGNED_DATA_GATEWAY_URL = local.http_signed_data_gateway_url
     AIRNODE_WALLET_PRIVATE_KEY   = var.airnode_wallet_private_key
   }
 
@@ -115,7 +115,7 @@ module "startCoordinatorOevGw" {
   secrets_file       = var.secrets_file
 
   environment_variables = {
-    OEV_GATEWAY_URL            = module.oevGw[0].api_url
+    OEV_GATEWAY_URL            = local.oev_gateway_url
     AIRNODE_WALLET_PRIVATE_KEY = var.airnode_wallet_private_key
   }
 
@@ -139,8 +139,8 @@ module "startCoordinatorHttpGwAndHttpSignedGw" {
   secrets_file       = var.secrets_file
 
   environment_variables = {
-    HTTP_GATEWAY_URL             = module.httpGw[0].api_url
-    HTTP_SIGNED_DATA_GATEWAY_URL = module.httpSignedGw[0].api_url
+    HTTP_GATEWAY_URL             = local.http_gateway_url
+    HTTP_SIGNED_DATA_GATEWAY_URL = local.http_signed_data_gateway_url
     AIRNODE_WALLET_PRIVATE_KEY   = var.airnode_wallet_private_key
   }
 
@@ -164,8 +164,8 @@ module "startCoordinatorHttpGwAndOevGw" {
   secrets_file       = var.secrets_file
 
   environment_variables = {
-    HTTP_GATEWAY_URL           = module.httpGw[0].api_url
-    OEV_GATEWAY_URL            = module.oevGw[0].api_url
+    HTTP_GATEWAY_URL           = local.http_gateway_url
+    OEV_GATEWAY_URL            = local.oev_gateway_url
     AIRNODE_WALLET_PRIVATE_KEY = var.airnode_wallet_private_key
   }
 
@@ -189,8 +189,8 @@ module "startCoordinatorHttpSignedGwAndOevGw" {
   secrets_file       = var.secrets_file
 
   environment_variables = {
-    HTTP_SIGNED_DATA_GATEWAY_URL = module.httpSignedGw[0].api_url
-    OEV_GATEWAY_URL              = module.oevGw[0].api_url
+    HTTP_SIGNED_DATA_GATEWAY_URL = local.http_signed_data_gateway_url
+    OEV_GATEWAY_URL              = local.oev_gateway_url
     AIRNODE_WALLET_PRIVATE_KEY   = var.airnode_wallet_private_key
   }
 
@@ -214,9 +214,9 @@ module "startCoordinatorAllGws" {
   secrets_file       = var.secrets_file
 
   environment_variables = {
-    HTTP_GATEWAY_URL             = module.httpGw[0].api_url
-    HTTP_SIGNED_DATA_GATEWAY_URL = module.httpSignedGw[0].api_url
-    OEV_GATEWAY_URL              = module.oevGw[0].api_url
+    HTTP_GATEWAY_URL             = local.http_gateway_url
+    HTTP_SIGNED_DATA_GATEWAY_URL = local.http_signed_data_gateway_url
+    OEV_GATEWAY_URL              = local.oev_gateway_url
     AIRNODE_WALLET_PRIVATE_KEY   = var.airnode_wallet_private_key
   }
 

diff --git a/packages/airnode-deployer/terraform/aws/modules/function/main.tf b/packages/airnode-deployer/terraform/aws/modules/function/main.tf
@@ -54,7 +54,7 @@ resource "aws_lambda_function" "lambda" {
   environment {
     variables = merge(
       var.environment_variables,
-      fileexists(var.secrets_file) ? { for tuple in regexall("(.*?)=(.*)", file(var.secrets_file)) : tuple[0] => trim(tuple[1], "\"'") } : {},
+      local.secrets,
       { AIRNODE_CLOUD_PROVIDER = "aws" }
     )
   }

diff --git a/packages/airnode-deployer/terraform/aws/modules/function/variables.tf b/packages/airnode-deployer/terraform/aws/modules/function/variables.tf
@@ -2,6 +2,19 @@ locals {
   uuid        = uuid()
   tmp_dir     = "/tmp/${var.name}#${local.uuid}"
   tmp_archive = "/tmp/${var.name}#${local.uuid}.zip"
+
+  # Secrets are already validated by the validator before reaching Terraform recipes. Any edge-cases that come to mind are most likely handled there.
+  #
+  # Read file and split it line by line. Using regex to avoid UNIX/Windows line-ending problems
+  secrets_lines = fileexists(var.secrets_file) ? regexall(".*", file(var.secrets_file)) : []
+  # Trim whitespaces from the line
+  secrets_lines_trimmed = [for line in local.secrets_lines : trimspace(line)]
+  # Discard commented lines (starting with '#')
+  secrets_lines_uncommented = [for line in local.secrets_lines_trimmed : line if !startswith(line, "#")]
+  # Discard lines not matching the pattern and split them. We're looking for line that has non-whitespace characters before '=' and anything after
+  secrets_lines_matched = [for line in local.secrets_lines_uncommented : regex("^([^[:space:]]+?)=(.*)$", line) if can(regex("^([^[:space:]]+?)=(.*)$", line))]
+  # Convert the list to a map, remove quotation marks around the values. When duplicate keys are encountered the last found value is used.
+  secrets = merge([for tuple in local.secrets_lines_matched : { (tuple[0]) = trim(tuple[1], "\"'") }]...)
 }
 
 variable "handler" {

diff --git a/packages/airnode-deployer/terraform/aws/outputs.tf b/packages/airnode-deployer/terraform/aws/outputs.tf
@@ -1,11 +1,11 @@
 output "http_gateway_url" {
-  value = var.http_gateway_enabled == false ? null : "${module.httpGw[0].api_url}/${random_uuid.http_path_key.result}"
+  value = local.http_gateway_url
 }
 
 output "http_signed_data_gateway_url" {
-  value = var.http_signed_data_gateway_enabled == false ? null : "${module.httpSignedGw[0].api_url}/${random_uuid.http_signed_data_path_key.result}"
+  value = local.http_signed_data_gateway_url
 }
 
 output "oev_gateway_url" {
-  value = var.oev_gateway_enabled == false ? null : "${module.oevGw[0].api_url}/${random_uuid.oev_path_key.result}"
+  value = local.oev_gateway_url
 }
diff --git a/packages/airnode-deployer/terraform/aws/variables.tf b/packages/airnode-deployer/terraform/aws/variables.tf
@@ -6,6 +6,10 @@ locals {
   # deployment_id - 11 characters
   # dash between - 1 character
   name_prefix = "${var.infrastructure_name}-${var.deployment_id}"
+
+  http_gateway_url             = var.http_gateway_enabled == false ? null : "${module.httpGw[0].api_url}/${random_uuid.http_path_key.result}"
+  http_signed_data_gateway_url = var.http_signed_data_gateway_enabled == false ? null : "${module.httpSignedGw[0].api_url}/${random_uuid.http_signed_data_path_key.result}"
+  oev_gateway_url              = var.oev_gateway_enabled == false ? null : "${module.oevGw[0].api_url}/${random_uuid.oev_path_key.result}"
 }
 
 variable "aws_region" {

diff --git a/packages/airnode-deployer/terraform/gcp/main.tf b/packages/airnode-deployer/terraform/gcp/main.tf
@@ -76,9 +76,9 @@ module "startCoordinator" {
   airnode_bucket        = var.airnode_bucket
   deployment_bucket_dir = var.deployment_bucket_dir
   environment_variables = {
-    HTTP_GATEWAY_URL             = var.http_gateway_enabled ? "https://${module.httpGw[0].api_url}/${random_uuid.http_path_key.result}" : null
-    HTTP_SIGNED_DATA_GATEWAY_URL = var.http_signed_data_gateway_enabled ? "https://${module.httpSignedGw[0].api_url}${random_uuid.http_signed_data_path_key.result}" : null
-    OEV_GATEWAY_URL              = var.oev_gateway_enabled ? "https://${module.oevGw[0].api_url}${random_uuid.oev_path_key.result}" : null
+    HTTP_GATEWAY_URL             = local.http_gateway_url
+    HTTP_SIGNED_DATA_GATEWAY_URL = local.http_signed_data_gateway_url
+    OEV_GATEWAY_URL              = local.oev_gateway_url
     AIRNODE_WALLET_PRIVATE_KEY   = var.airnode_wallet_private_key
   }
 

diff --git a/packages/airnode-deployer/terraform/gcp/modules/function/main.tf b/packages/airnode-deployer/terraform/gcp/modules/function/main.tf
@@ -67,7 +67,7 @@ resource "google_cloudfunctions_function" "function" {
   max_instances         = var.max_instances
   environment_variables = merge(
     var.environment_variables,
-    fileexists(var.secrets_file) ? { for tuple in regexall("(.*?)=(.*)", file(var.secrets_file)) : tuple[0] => trim(tuple[1], "\"'") } : {},
+    local.secrets,
     { AIRNODE_CLOUD_PROVIDER = "gcp" }
   )
   service_account_email = google_service_account.function_service_account.email

diff --git a/packages/airnode-deployer/terraform/gcp/modules/function/variables.tf b/packages/airnode-deployer/terraform/gcp/modules/function/variables.tf
@@ -6,6 +6,19 @@ locals {
   # are called europe-west1 and us-central1, respectively, elsewhere in Google documentation.
   # https://cloud.google.com/appengine/docs/locations
   app_engine_location_id = var.region == "europe-west1" ? "europe-west" : (var.region == "us-central1" ? "us-central" : var.region)
+
+  # Secrets are already validated by the validator before reaching Terraform recipes. Any edge-cases that come to mind are most likely handled there.
+  #
+  # Read file and split it line by line. Using regex to avoid UNIX/Windows line-ending problems
+  secrets_lines = fileexists(var.secrets_file) ? regexall(".*", file(var.secrets_file)) : []
+  # Trim whitespaces from the line
+  secrets_lines_trimmed = [for line in local.secrets_lines : trimspace(line)]
+  # Discard commented lines (starting with '#')
+  secrets_lines_uncommented = [for line in local.secrets_lines_trimmed : line if !startswith(line, "#")]
+  # Discard lines not matching the pattern and split them. We're looking for line that has non-whitespace characters before '=' and anything after
+  secrets_lines_matched = [for line in local.secrets_lines_uncommented : regex("^([^[:space:]]+?)=(.*)$", line) if can(regex("^([^[:space:]]+?)=(.*)$", line))]
+  # Convert the list to a map, remove quotation marks around the values. When duplicate keys are encountered the last found value is used.
+  secrets = merge([for tuple in local.secrets_lines_matched : { (tuple[0]) = trim(tuple[1], "\"'") }]...)
 }
 
 variable "entry_point" {

diff --git a/packages/airnode-deployer/terraform/gcp/outputs.tf b/packages/airnode-deployer/terraform/gcp/outputs.tf
@@ -1,11 +1,11 @@
 output "http_gateway_url" {
-  value = var.http_gateway_enabled == false ? null : "https://${module.httpGw[0].api_url}/${random_uuid.http_path_key.result}"
+  value = local.http_gateway_url
 }
 
 output "http_signed_data_gateway_url" {
-  value = var.http_signed_data_gateway_enabled == false ? null : "https://${module.httpSignedGw[0].api_url}/${random_uuid.http_signed_data_path_key.result}"
+  value = local.http_signed_data_gateway_url
 }
 
 output "oev_gateway_url" {
-  value = var.oev_gateway_enabled == false ? null : "https://${module.oevGw[0].api_url}/${random_uuid.oev_path_key.result}"
+  value = local.oev_gateway_url
 }
diff --git a/packages/airnode-deployer/terraform/gcp/variables.tf b/packages/airnode-deployer/terraform/gcp/variables.tf
@@ -6,6 +6,10 @@ locals {
   # deployment_id - 11 characters
   # dash between - 1 character
   name_prefix = "${var.infrastructure_name}-${var.deployment_id}"
+
+  http_gateway_url             = var.http_gateway_enabled == false ? null : "https://${module.httpGw[0].api_url}/${random_uuid.http_path_key.result}"
+  http_signed_data_gateway_url = var.http_signed_data_gateway_enabled == false ? null : "https://${module.httpSignedGw[0].api_url}/${random_uuid.http_signed_data_path_key.result}"
+  oev_gateway_url              = var.oev_gateway_enabled == false ? null : "https://${module.oevGw[0].api_url}/${random_uuid.oev_path_key.result}"
 }
 
 variable "gcp_project" {