AWS EKS cluster autoscaler implementation (#460)

* AWS EKS cluster autoscaler implementation * terraform syntax formatted * terraform syntax formatted Co-authored-by: Sergey Yelenski <[email protected]>
arriven · Mar 30, 2022 · 7033aa2 · 7033aa2
1 parent 7dbb46f
commit 7033aa2
Show file tree

Hide file tree

Showing 23 changed files with 1,231 additions and 0 deletions.
diff --git a/terraform/aws/eks-cluster/data.tf b/terraform/aws/eks-cluster/data.tf
@@ -0,0 +1 @@
+data "aws_caller_identity" "current" {}
diff --git a/terraform/aws/eks-cluster/main.tf b/terraform/aws/eks-cluster/main.tf
@@ -0,0 +1,59 @@
+# Define local variables
+locals {
+  cluster_version = "1.21"
+  cluster_name    = "eks-${var.project}"
+}
+
+# Create VPC network
+module "network" {
+  source = "./modules/network"
+
+  name = "vpc-${var.project}"
+  tags = {
+    "kubernetes.io/cluster/eks-${var.project}" = "shared"
+  }
+}
+
+# Create EKS cluster
+module "eks_cluster" {
+  source = "./modules/eks-cluster"
+
+  cluster_name           = local.cluster_name
+  cluster_version        = local.cluster_version
+  vpc_id                 = module.network.vpc_id
+  subnets                = module.network.private_subnet_ids
+  source_security_groups = module.eks_nodes.security_group_id
+}
+
+# Create EKS nodes
+module "eks_nodes" {
+  source = "./modules/eks-nodes"
+
+  autoscale_group_name   = "eks-${var.project}-node"
+  cluster_name           = local.cluster_name
+  cluster_version        = local.cluster_version
+  cluster_endpoint       = module.eks_cluster.endpoint
+  cluster_ca_data        = module.eks_cluster.ca_data
+  vpc_id                 = module.network.vpc_id
+  subnets                = module.network.private_subnet_ids
+  source_security_groups = module.eks_cluster.security_group_id
+  key_name               = var.key_name
+  public_key             = file(var.public_key)
+  instance_type          = var.eks_node_instance_type
+  desired_capacity       = var.eks_node_desired_capacity
+  max_size               = var.eks_node_max_size
+  min_size               = var.eks_node_min_size
+}
+
+# Setup kubernetes
+module "kubernetes" {
+  source = "./modules/kubernetes"
+
+  profile                  = var.profile
+  vpc_id                   = module.network.vpc_id
+  cluster_name             = local.cluster_name
+  cluster_endpoint         = module.eks_cluster.endpoint
+  cluster_ca_data          = module.eks_cluster.ca_data
+  worker_node_iam_role_arn = module.eks_nodes.worker_node_iam_role_arn
+  autoscaler_iam_role_arn  = module.eks_cluster.autoscaler_iam_role_arn
+}
diff --git a/terraform/aws/eks-cluster/modules/eks-cluster/data.tf b/terraform/aws/eks-cluster/modules/eks-cluster/data.tf
@@ -0,0 +1,47 @@
+data "tls_certificate" "cert" {
+  url = aws_eks_cluster.eks_cluster.identity[0].oidc[0].issuer
+}
+
+data "aws_iam_policy_document" "eks_cluster_assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["eks.amazonaws.com"]
+    }
+  }
+}
+
+# Cluster autoscaler
+data "aws_iam_policy_document" "assume_role_policy_web_identity" {
+  statement {
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+
+    principals {
+      type        = "Federated"
+      identifiers = ["${aws_iam_openid_connect_provider.oidc_provider.id}"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "${replace(aws_iam_openid_connect_provider.oidc_provider.url, "https://", "")}:sub"
+      values   = ["system:serviceaccount:kube-system:aws-node", "system:serviceaccount:kube-system:cluster-autoscaler"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "eks_cluster_autoscaler_policy" {
+  statement {
+    resources = ["*"]
+    actions = [
+      "autoscaling:DescribeAutoScalingGroups",
+      "autoscaling:DescribeAutoScalingInstances",
+      "autoscaling:DescribeLaunchConfigurations",
+      "autoscaling:DescribeTags",
+      "autoscaling:SetDesiredCapacity",
+      "autoscaling:TerminateInstanceInAutoScalingGroup",
+      "ec2:DescribeLaunchTemplateVersions"
+    ]
+  }
+}
diff --git a/terraform/aws/eks-cluster/modules/eks-cluster/main.tf b/terraform/aws/eks-cluster/modules/eks-cluster/main.tf
@@ -0,0 +1,107 @@
+# Create EKS cluster role
+resource "aws_iam_role" "eks_cluster" {
+  name = "AWSEKSClusterRole"
+  # name = "AmazonEKSClusterRole"
+  assume_role_policy = data.aws_iam_policy_document.eks_cluster_assume_role_policy.json
+
+  tags = {
+    Cluster = var.cluster_name
+  }
+
+  lifecycle {
+    ignore_changes = [name, name_prefix]
+  }
+}
+
+# Create EKS cluster role policy attachment
+resource "aws_iam_role_policy_attachment" "eks_cluster_AmazonEKSClusterPolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
+  role       = aws_iam_role.eks_cluster.name
+}
+
+resource "aws_iam_role_policy_attachment" "eks_cluster_AmazonEKSServicePolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
+  role       = aws_iam_role.eks_cluster.name
+}
+
+# Create EKS cluster autoscaler role
+resource "aws_iam_role" "eks_cluster_autoscaler" {
+  name               = "AmazonEKSClusterAutoscalerRole"
+  assume_role_policy = data.aws_iam_policy_document.assume_role_policy_web_identity.json
+
+  inline_policy {
+    name   = "AmazonEKSClusterAutoscalerPolicy"
+    policy = data.aws_iam_policy_document.eks_cluster_autoscaler_policy.json
+  }
+
+  tags = {
+    Cluster = var.cluster_name
+  }
+
+  lifecycle {
+    ignore_changes = [name, name_prefix]
+  }
+
+  depends_on = [aws_iam_openid_connect_provider.oidc_provider]
+}
+
+# Create EKS cluster
+resource "aws_eks_cluster" "eks_cluster" {
+  name                      = var.cluster_name
+  version                   = var.cluster_version
+  role_arn                  = aws_iam_role.eks_cluster.arn
+  enabled_cluster_log_types = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
+
+  vpc_config {
+    security_group_ids = [aws_security_group.control_plane.id]
+    subnet_ids         = var.subnets
+  }
+
+  depends_on = [
+    aws_iam_role_policy_attachment.eks_cluster_AmazonEKSClusterPolicy,
+    aws_iam_role_policy_attachment.eks_cluster_AmazonEKSServicePolicy
+  ]
+}
+
+# create OIDC provider
+resource "aws_iam_openid_connect_provider" "oidc_provider" {
+  client_id_list  = ["sts.amazonaws.com"]
+  thumbprint_list = [data.tls_certificate.cert.certificates[0].sha1_fingerprint]
+  url             = aws_eks_cluster.eks_cluster.identity[0].oidc[0].issuer
+}
+
+# Create EKS cluster security group
+resource "aws_security_group" "control_plane" {
+  name        = "${var.cluster_name}-control-plane"
+  description = "Cluster communication with worker nodes"
+  vpc_id      = var.vpc_id
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  revoke_rules_on_delete = true
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = {
+    Name    = "${var.cluster_name}-control-plane"
+    Cluster = var.cluster_name
+  }
+}
+
+# Create EKS cluster security group rules
+resource "aws_security_group_rule" "control_plane_ingress_nodes" {
+  description              = "Allow cluster control plane to receive communication from the worker Kubelets"
+  type                     = "ingress"
+  from_port                = 443
+  to_port                  = 443
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.control_plane.id
+  source_security_group_id = var.source_security_groups
+}
diff --git a/terraform/aws/eks-cluster/modules/eks-cluster/outputs.tf b/terraform/aws/eks-cluster/modules/eks-cluster/outputs.tf
@@ -0,0 +1,17 @@
+output "endpoint" {
+  value = aws_eks_cluster.eks_cluster.endpoint
+}
+
+output "ca_data" {
+  value = aws_eks_cluster.eks_cluster.certificate_authority.0.data
+}
+
+output "security_group_id" {
+  description = "EKS Control Plane Security group ID"
+  value       = aws_security_group.control_plane.id
+}
+
+output "autoscaler_iam_role_arn" {
+  description = "EKS cluster autoscaler role ARN"
+  value       = aws_iam_role.eks_cluster_autoscaler.arn
+}
diff --git a/terraform/aws/eks-cluster/modules/eks-cluster/variables.tf b/terraform/aws/eks-cluster/modules/eks-cluster/variables.tf
@@ -0,0 +1,21 @@
+variable "vpc_id" {
+  description = "VPC ID"
+}
+
+variable "cluster_name" {
+  description = "EKS cluster name"
+}
+
+variable "cluster_version" {
+  description = "EKS cluster version"
+}
+
+variable "source_security_groups" {
+  description = "A list of source security groups which can connect to the EKS cluster"
+}
+
+variable "subnets" {
+  description = "A list of subnets to place the EKS cluster"
+  type        = list(string)
+}
+
diff --git a/terraform/aws/eks-cluster/modules/eks-nodes/data.tf b/terraform/aws/eks-cluster/modules/eks-nodes/data.tf
@@ -0,0 +1,46 @@
+data "aws_iam_policy_document" "eks_worker_node_assume_role_policy" {
+  statement {
+    sid = "EKSWorkerAssumeRole"
+
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ec2.amazonaws.com"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "assume_role_policy" {
+  statement {
+    actions   = ["sts:AssumeRole"]
+    resources = ["*"]
+  }
+}
+
+data "aws_ami" "amazon_eks_nodes" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["amazon-eks-node-${var.cluster_version}-*"]
+  }
+
+  filter {
+    name = "owner-alias"
+    values = [
+      "amazon",
+    ]
+  }
+}
+
+data "template_file" "user_data" {
+  template = file("${path.root}/scripts/node-user-data.sh")
+
+  vars = {
+    CLUSTER_NAME     = var.cluster_name
+    CLUSTER_ENDPOINT = var.cluster_endpoint
+    CLUSTER_CA_DATA  = var.cluster_ca_data
+  }
+}