Configurable IAM policies for ECS distrib indexers (#4897)

* Configurable IAM policies for ECS distrib indexers * Add some docs about the new var * Apply review wording suggestion Co-authored-by: Adrien Guillo <[email protected]> --------- Co-authored-by: Adrien Guillo <[email protected]>
quickwit-oss · Apr 24, 2024 · 6f26989 · 6f26989
1 parent 8d419c8
commit 6f26989
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 12 deletions.
diff --git a/distribution/ecs/README.md b/distribution/ecs/README.md
@@ -55,6 +55,10 @@ You can use sidecars to inject additional secrets as files. This can be
 useful for configuring sources such as Kafka. See `./example/kafka.tf` for an
 example.
 
+To access external AWS services like the Kinesis source, use the
+`quickwit_indexer.extra_task_policy_arns` variable to attach the necessary
+IAM policies to indexers.
+
 ## Running the example stack
 
 We provide an example of self contained deployment with an ad-hoc VPC. 

diff --git a/distribution/ecs/example/terraform.tf b/distribution/ecs/example/terraform.tf
@@ -41,6 +41,7 @@ module "quickwit" {
   #   memory                = 8192
   #   cpu                   = 4096
   #   ephemeral_storage_gib = 50
+  #   extra_task_policy_arns = ["arn:aws:iam::aws:policy/AmazonKinesisFullAccess"]
   # }
 
   # quickwit_metastore = {
@@ -53,7 +54,6 @@ module "quickwit" {
   #   desired_count         = 1
   #   memory                = 2048
   #   cpu                   = 1024
-  #   ephemeral_storage_gib = 21
   # }
 
   # quickwit_control_plane = {

diff --git a/distribution/ecs/quickwit/service/config.tf b/distribution/ecs/quickwit/service/config.tf
@@ -28,4 +28,7 @@ locals {
     },
   ]
 
+  nb_extra_policies             = length(var.service_config.extra_task_policy_arns)
+  extra_tasks_iam_role_policies = { for i in range(local.nb_extra_policies) : "extra_policy_${i}" => var.service_config.extra_task_policy_arns[i] }
+  tasks_iam_role_policies       = merge({ s3_access = var.s3_access_policy_arn }, local.extra_tasks_iam_role_policies)
 }
diff --git a/distribution/ecs/quickwit/service/ecs.tf b/distribution/ecs/quickwit/service/ecs.tf
@@ -128,9 +128,7 @@ module "quickwit_service" {
     var.postgres_credential_arn
   ]
 
-  tasks_iam_role_policies = {
-    s3_access = var.s3_access_policy_arn
-  }
+  tasks_iam_role_policies = local.tasks_iam_role_policies
 
   task_exec_iam_role_policies = {
     policy = var.task_execution_policy_arn

diff --git a/distribution/ecs/quickwit/service/variables.tf b/distribution/ecs/quickwit/service/variables.tf
@@ -38,10 +38,11 @@ variable "quickwit_image" {}
 
 variable "service_config" {
   type = object({
-    desired_count         = optional(number, 1)
-    memory                = number
-    cpu                   = number
-    ephemeral_storage_gib = optional(number, 21)
+    desired_count          = optional(number, 1)
+    memory                 = number
+    cpu                    = number
+    ephemeral_storage_gib  = optional(number, 21)
+    extra_task_policy_arns = optional(list(string), [])
   })
 }
 

diff --git a/distribution/ecs/quickwit/variables.tf b/distribution/ecs/quickwit/variables.tf
@@ -72,10 +72,11 @@ variable "log_configuration" {
 variable "quickwit_indexer" {
   description = "Indexer service sizing configurations"
   type = object({
-    desired_count         = optional(number, 1)
-    memory                = optional(number, 4096)
-    cpu                   = optional(number, 1024)
-    ephemeral_storage_gib = optional(number, 21)
+    desired_count          = optional(number, 1)
+    memory                 = optional(number, 4096)
+    cpu                    = optional(number, 1024)
+    ephemeral_storage_gib  = optional(number, 21)
+    extra_task_policy_arns = optional(list(string), [])
   })
   default = {}
 }