rework custom networking

main.tf

@@ -15,6 +15,11 @@
 */
 
 locals {
+  eni_configs = [for e in var.pods_subnets : {
+    name           = e.availability_zone
+    subnet         = e.id
+    securityGroups = [aws_eks_cluster.quortex.vpc_config[0].cluster_security_group_id]
+  }]
   # The Quortex cluster OIDC issuer.
   cluster_oidc_issuer = trimprefix(aws_eks_cluster.quortex.identity[0].oidc[0].issuer, "https://")
   node_group_labels = [
@@ -60,6 +65,19 @@ data "aws_caller_identity" "current" {}
 # This datasource is used to get the region currently used by the AWS provider
 data "aws_region" "current" {}
 
+provider "helm" {
+  kubernetes {
+    host                   = aws_eks_cluster.quortex.endpoint
+    cluster_ca_certificate = base64decode(aws_eks_cluster.quortex.certificate_authority[0].data)
+    exec {
+      api_version = "client.authentication.k8s.io/v1beta1"
+      # This requires the awscli to be installed locally where Terraform is executed
+      command = "aws"
+      args    = ["eks", "get-token", "--cluster-name", aws_eks_cluster.quortex.name]
+    }
+  }
+}
+
 # Cluster
 resource "aws_eks_cluster" "quortex" {
   name     = var.cluster_name
@@ -170,13 +188,6 @@ locals {
   }
 }
 
-# delays creation of add-ons after aws_eks_cluster
-resource "time_sleep" "wait_3_minutes" {
-  depends_on = [aws_eks_cluster.quortex]
-
-  create_duration = "3m"
-}
-
 # Eks addons
 resource "aws_eks_addon" "quortex_addon" {
   for_each = { for k, v in var.cluster_addons : k => v }
@@ -256,3 +267,16 @@ resource "aws_cloudwatch_log_group" "cluster_logs" {
   retention_in_days = var.cluster_logs_retention
   tags              = var.tags
 }
+
+resource "helm_release" "eni_configs" {
+  version    = "1.0.0"
+  chart      = "empty"
+  repository = "https://quortex.github.io/helm-charts"
+  name       = "aws-vpc-cni-config"
+
+  values = [
+    templatefile("${path.module}/values.yaml", {
+      eniConfigs : jsonencode(local.eni_configs)
+    })
+  ]
+}

values.yaml

@@ -0,0 +1,16 @@
+manifests: |-
+  {{- range .Values.eniConfigs }}
+  apiVersion: crd.k8s.amazonaws.com/v1alpha1
+  kind: ENIConfig
+  metadata:
+    name: {{ .name }}
+  spec:
+    subnet: {{ .subnet }}
+    {{- with .securityGroups }}
+    securityGroups:
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  ---
+  {{- end }}
+
+eniConfigs: ${eniConfigs}

variables.tf

@@ -147,6 +147,15 @@ variable "master_authorized_networks" {
   default     = {}
 }
 
+variable "pods_subnets" {
+  type        = map(object({ id = string, availability_zone = string, cidr = string, public = bool }))
+  description = <<EOT
+A map representing the pods subnets. Each item contains the subnet's ID,
+Availability Zone, cidr block, and whether the subnet is public or not.
+EOT
+  default     = {}
+}
+
 variable "tags" {
   type        = map(any)
   description = "The EKS resource tags (a map of key/value pairs) to be applied to the cluster."

versions.tf

@@ -22,6 +22,10 @@ terraform {
       source  = "hashicorp/aws"
       version = ">=5.0.0"
     }
+    helm = {
+      source  = "hashicorp/helm"
+      version = ">=2.0.0"
+    }
     kubernetes = {
       source  = "hashicorp/kubernetes"
       version = ">=2.0.0"
@@ -34,9 +38,5 @@ terraform {
       source  = "hashicorp/tls"
       version = ">=3.4.0"
     }
-    time = {
-      source  = "hashicorp/time"
-      version = ">=0.11.1"
-    }
   }
 }