From a6e4be4d1b3e8add62f57ee8b6f8f16ad8c549e4 Mon Sep 17 00:00:00 2001 From: Mateusz Nowakowski Date: Fri, 13 Dec 2024 23:48:44 +0100 Subject: [PATCH] feat: add more configuration to EKS deployment --- k8s/apps/echoserver/scripts/deploy-on-eks.sh | 26 ++++ terraform/aws/aws-eks/README.md | 7 +- .../module/cluster-config-chart/.helmignore | 23 ++++ .../module/cluster-config-chart/Chart.yaml | 23 ++++ .../module/cluster-config-chart/Makefile | 10 ++ .../cluster-config-chart/templates/NOTES.txt | 4 + .../templates/_helpers.tpl | 51 ++++++++ .../templates/ingress-classes.yaml | 41 ++++++ .../templates/netpol.yaml | 11 ++ .../templates/node-pools.yaml | 44 +++++++ .../templates/storage-classes.yaml | 20 +++ .../module/cluster-config-chart/values.yaml | 2 + .../aws/aws-eks/module/configure-cluster.sh | 30 +++++ terraform/aws/aws-eks/module/eks-addons.tf | 118 ++++++++++++++++++ terraform/aws/aws-eks/module/eks.tf | 25 +++- .../module/namespace-config-chart/.helmignore | 23 ++++ .../module/namespace-config-chart/Chart.yaml | 23 ++++ .../module/namespace-config-chart/Makefile | 10 ++ .../templates/NOTES.txt | 1 + .../templates/_helpers.tpl | 51 ++++++++ .../templates/limitrange.yaml | 26 ++++ .../templates/netpol.yaml | 0 .../templates/quota.yaml | 27 ++++ .../namespace-config-chart/templates/sa.yaml | 30 +++++ .../module/namespace-config-chart/values.yaml | 10 ++ .../module/pod-identity-associations.tf | 6 + terraform/aws/aws-eks/module/variables.tf | 33 +++++ terraform/aws/aws-eks/module/versions.tf | 4 + .../aws/aws-eks/stage/dev/terragrunt.hcl | 14 +++ terraform/aws/aws-eks/test/deployment.yaml | 2 +- terraform/aws/aws-eks/test/ingress.yaml | 4 +- terraform/aws/aws-eks/test/ingressclass.yaml | 24 ---- terraform/aws/aws-eks/test/service.yaml | 2 +- 33 files changed, 693 insertions(+), 32 deletions(-) create mode 100644 k8s/apps/echoserver/scripts/deploy-on-eks.sh create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/.helmignore create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/Chart.yaml create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/Makefile create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/templates/NOTES.txt create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/templates/_helpers.tpl create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/templates/ingress-classes.yaml create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/templates/netpol.yaml create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/templates/node-pools.yaml create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/templates/storage-classes.yaml create mode 100644 terraform/aws/aws-eks/module/cluster-config-chart/values.yaml create mode 100755 terraform/aws/aws-eks/module/configure-cluster.sh create mode 100644 terraform/aws/aws-eks/module/eks-addons.tf create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/.helmignore create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/Chart.yaml create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/Makefile create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/templates/NOTES.txt create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/templates/_helpers.tpl create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/templates/limitrange.yaml create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/templates/netpol.yaml create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/templates/quota.yaml create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/templates/sa.yaml create mode 100644 terraform/aws/aws-eks/module/namespace-config-chart/values.yaml create mode 100644 terraform/aws/aws-eks/module/pod-identity-associations.tf delete mode 100644 terraform/aws/aws-eks/test/ingressclass.yaml diff --git a/k8s/apps/echoserver/scripts/deploy-on-eks.sh b/k8s/apps/echoserver/scripts/deploy-on-eks.sh new file mode 100644 index 00000000..b61dfadc --- /dev/null +++ b/k8s/apps/echoserver/scripts/deploy-on-eks.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" + +CN="echoserver.learning.testing.minikube" + +kubectl config use-context minikube || echo "Minikube not present in kube context" && { + kubectl create ns learning &>/dev/null + kubectl config set-context --current --namespace learning + [ -e "/tmp/${CN}.key" ] || { + openssl req -x509 -sha256 -nodes -days 365 -subj "/CN=${CN}" -newkey rsa:2048 -keyout "/tmp/${CN}.key" -out "/tmp/${CN}.crt" + } + helm upgrade --install echoserver "$(dirname "${SCRIPT_DIR}")" -n learning \ + --set ingress.tls.crt="$(base64 -w 0 /tmp/${CN}.crt)" \ + --set ingress.tls.key="$(base64 -w 0 /tmp/${CN}.key)" \ + --set ingress.host="${CN}" \ + --set ingress.class=nginx \ + --set networkPolicy.enabled=true + + # while [ -z "$(kubectl get ingress echoserver -n learning -o jsonpath="{.status..ip}" | xargs)" ]; do + # sleep 1 + # echo "Awaiting for LoadBalancer for Ingress..." + # done + # INGRESS_IP="$(kubectl get ingress echoserver -n learning -o jsonpath="{.status..ip}")" + # CHANGED=$(grep -c "${INGRESS_IP} ${CN}" /etc/hosts) + # [ "${CHANGED}" -eq 0 ] && echo "update hosts" && sudo -E sh -c "echo \"${INGRESS_IP} ${CN}\" >> /etc/hosts" || echo "hosts already present" +} diff --git a/terraform/aws/aws-eks/README.md b/terraform/aws/aws-eks/README.md index a545fa61..d3b8653f 100644 --- a/terraform/aws/aws-eks/README.md +++ b/terraform/aws/aws-eks/README.md @@ -1,10 +1,13 @@ # Terraform :: EKS deployment -Terraform scripts deploy ROSA +Terraform scripts deploy EKS In particular it creates: - EKS with Auto mode +- EFS and CSI Snapshots addons +- Configure storage, ingress, node pool classes +- Configure app namespaces (quuota, limits, networkpolicies) ## Prerequisites @@ -12,7 +15,7 @@ In particular it creates: - Latest Terraform or OpenTofu, Terragrunt installed -- jq tool installed +- jq, kubectl tools installed - EKS CTL CLI installed (optionally): diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/.helmignore b/terraform/aws/aws-eks/module/cluster-config-chart/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/Chart.yaml b/terraform/aws/aws-eks/module/cluster-config-chart/Chart.yaml new file mode 100644 index 00000000..4638e7f9 --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: cluster-config +description: A Helm chart for EKS configuration + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.31.0 diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/Makefile b/terraform/aws/aws-eks/module/cluster-config-chart/Makefile new file mode 100644 index 00000000..1d6cff98 --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/Makefile @@ -0,0 +1,10 @@ +deploy: ## deploy EKS configuration + helm upgrade --install cluster-config --namespace=cluster-config --create-namespace . + +undeploy: ## undeploys EKS configuration + helm uninstall cluster-config --namespace=cluster-config + +help: ## show usage and tasks (default) + @eval $$(sed -E -n 's/^([\*\.a-zA-Z0-9_-]+):.*?## (.*)$$/printf "\\033[36m%-30s\\033[0m %s\\n" "\1" "\2" ;/; ta; b; :a p' $(MAKEFILE_LIST)) +.DEFAULT_GOAL := help +.PHONY: deploy undeploy diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/templates/NOTES.txt b/terraform/aws/aws-eks/module/cluster-config-chart/templates/NOTES.txt new file mode 100644 index 00000000..4917e731 --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/templates/NOTES.txt @@ -0,0 +1,4 @@ +Install cluster EKS configurations: +- Compute Node Pool +- Storage Classe +- ALB Ingress Classes diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/templates/_helpers.tpl b/terraform/aws/aws-eks/module/cluster-config-chart/templates/_helpers.tpl new file mode 100644 index 00000000..f9bbdb38 --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-config.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-config.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-config.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-config.labels" -}} +helm.sh/chart: {{ include "cluster-config.chart" . }} +{{ include "cluster-config.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-config.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-config.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/templates/ingress-classes.yaml b/terraform/aws/aws-eks/module/cluster-config-chart/templates/ingress-classes.yaml new file mode 100644 index 00000000..adbefed8 --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/templates/ingress-classes.yaml @@ -0,0 +1,41 @@ +# yamllint disable-file +--- +apiVersion: eks.amazonaws.com/v1 +kind: IngressClassParams +metadata: + name: external-alb +spec: + scheme: internet-facing +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: external-alb +spec: + # Configures the IngressClass to use EKS Auto Mode + controller: eks.amazonaws.com/alb + parameters: + apiGroup: eks.amazonaws.com + kind: IngressClassParams + name: external-alb +--- +apiVersion: eks.amazonaws.com/v1 +kind: IngressClassParams +metadata: + name: internal-alb +spec: + scheme: internal +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + annotations: + ingressclass.kubernetes.io/is-default-class: "true" + name: internal-alb +spec: + # Configures the IngressClass to use EKS Auto Mode + controller: eks.amazonaws.com/alb + parameters: + apiGroup: eks.amazonaws.com + kind: IngressClassParams + name: internal-alb diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/templates/netpol.yaml b/terraform/aws/aws-eks/module/cluster-config-chart/templates/netpol.yaml new file mode 100644 index 00000000..5c352824 --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/templates/netpol.yaml @@ -0,0 +1,11 @@ +# yamllint disable-file +--- +# Enable AWS VPC CNI driver NetworkPolicies +apiVersion: v1 +kind: ConfigMap +metadata: + name: amazon-vpc-cni + namespace: kube-system +data: + enable-network-policy-controller: "true" +# NodePoolClasses then has to contain diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/templates/node-pools.yaml b/terraform/aws/aws-eks/module/cluster-config-chart/templates/node-pools.yaml new file mode 100644 index 00000000..53cf5000 --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/templates/node-pools.yaml @@ -0,0 +1,44 @@ +# yamllint disable-file +--- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: compute-spot-arm64 +spec: + disruption: + budgets: + - nodes: 10% + consolidateAfter: 30s + consolidationPolicy: WhenEmptyOrUnderutilized + template: + spec: + expireAfter: 336h + nodeClassRef: + group: eks.amazonaws.com + kind: NodeClass + name: default + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: + - spot + - key: eks.amazonaws.com/instance-category + operator: In + values: + - c + - m + - r + - key: eks.amazonaws.com/instance-generation + operator: Gt + values: + - "4" + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + terminationGracePeriod: 24h0m0s diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/templates/storage-classes.yaml b/terraform/aws/aws-eks/module/cluster-config-chart/templates/storage-classes.yaml new file mode 100644 index 00000000..6a42d97e --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/templates/storage-classes.yaml @@ -0,0 +1,20 @@ +# yamllint disable-file +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ebs-gp3 + annotations: + storageclass.kubernetes.io/is-default-class: "true" + labels: + {{- include "cluster-config.labels" . | nindent 4 }} +provisioner: ebs.csi.eks.amazonaws.com +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +parameters: + type: gp3 + csi.storage.k8s.io/fstype: xfs + #TODO https://docs.aws.amazon.com/eks/latest/userguide/create-storage-class.html + # encrypted: "true" + # kmsKeyId: +allowVolumeExpansion: true diff --git a/terraform/aws/aws-eks/module/cluster-config-chart/values.yaml b/terraform/aws/aws-eks/module/cluster-config-chart/values.yaml new file mode 100644 index 00000000..fa2759cd --- /dev/null +++ b/terraform/aws/aws-eks/module/cluster-config-chart/values.yaml @@ -0,0 +1,2 @@ +clusterName: dev-us-east-1 +region: us-east-1 diff --git a/terraform/aws/aws-eks/module/configure-cluster.sh b/terraform/aws/aws-eks/module/configure-cluster.sh new file mode 100755 index 00000000..3aba3d6c --- /dev/null +++ b/terraform/aws/aws-eks/module/configure-cluster.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +CLUSTER_NAME="${1:?CLUSTER_NAME is required}" +REGION="${2:?REGION is required}" +NAMESPACES="${3:?NAMESPACES is required}" + +set -e +set -x + +DIRNAME="$(dirname "$0")" + +# login to AWS - assuming running as IAM principal being registered via aws_eks_access_entry and aws_eks_access_policy_association +aws eks update-kubeconfig --name "${CLUSTER_NAME}" --region "${REGION}" + +helm upgrade --install cluster-config -n cluster-config --create-namespace "${DIRNAME}/cluster-config-chart" \ + --set clusterName="${CLUSTER_NAME}" \ + --set region="${REGION}" + +for NAMESPACE in $(echo "${NAMESPACES}" | jq -cr '.[]'); do + + NS="$(echo "${NAMESPACE}" | jq -r ".name")" + QUOTA="$(echo "${NAMESPACE}" | jq -r ".quota")" + + [ -n "$(kubectl get ns "${NS}" --no-headers --ignore-not-found)" ] || { + kubectl create ns "${NS}" + } + helm upgrade --install "ns-${NS}-config" -n cluster-config --create-namespace "${DIRNAME}/namespace-config-chart" \ + --set namespace="${NS}" \ + --set-json quota="$(echo "${QUOTA}" | jq -r)" +done diff --git a/terraform/aws/aws-eks/module/eks-addons.tf b/terraform/aws/aws-eks/module/eks-addons.tf new file mode 100644 index 00000000..6a0df331 --- /dev/null +++ b/terraform/aws/aws-eks/module/eks-addons.tf @@ -0,0 +1,118 @@ +# The following addons are preinstaled and managed by EKS Auto Mode automatically: +# https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html#addon-consider-auto +# CoreDNS +# KubeProxy +# AWS Load Balancer Controller +# Karpenter +# AWS EBS CSI Driver +# EKS Pod Identity Agent https://docs.aws.amazon.com/eks/latest/userguide/pod-id-association.html + +resource "aws_eks_addon" "snapshot-controller" { + cluster_name = aws_eks_cluster.cluster.name + addon_name = "snapshot-controller" + # addon_version = "v8.1.0-eksbuild.2" + resolve_conflicts_on_create = "OVERWRITE" + + configuration_values = jsonencode( + { + "nodeSelector" : { + "karpenter.sh/nodepool" : "system" + } + } + ) +} + +# https://docs.aws.amazon.com/eks/latest/userguide/workloads-add-ons-available-eks.html#add-ons-aws-efs-csi-driver +resource "aws_eks_addon" "efs" { + cluster_name = aws_eks_cluster.cluster.name + addon_name = "aws-efs-csi-driver" + # addon_version = "v2.1.0-eksbuild.1" + resolve_conflicts_on_create = "OVERWRITE" + + pod_identity_association { + role_arn = aws_iam_role.efs-addon.arn + service_account = "efs-csi-controller-sa" + } + + configuration_values = jsonencode( + { + "controller" : { + "nodeSelector" : { + "karpenter.sh/nodepool" : "system" + }, + "tolerations" : [ + { + "key" : "CriticalAddonsOnly", + "operator" : "Exists" + }, + { + "effect" : "NoExecute", + "operator" : "Exists", + "tolerationSeconds" : 300 + } + ] + } + } + ) +} + + +resource "aws_iam_role" "efs-addon" { + name = "${local.prefix}-AmazonEKSPodIdentityAmazonEFSCSIDriverRole" + description = "Allows pods running in Amazon EKS cluster to access AWS resources." + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = ["sts:AssumeRole", "sts:TagSession"] + Effect = "Allow" + Principal = { + Service = "pods.eks.amazonaws.com" + } + }, + ] + }) +} + + + +resource "aws_iam_role_policy_attachment" "efs-addon_AmazonEFSCSIDriverPolicy" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" + role = aws_iam_role.efs-addon.name +} + +# TODO CloudWatch addons does not supoport Pod Identifies yet, only IRSA + +# # https://docs.aws.amazon.com/eks/latest/userguide/workloads-add-ons-available-eks.html#add-ons-aws-efs-csi-driver +# resource "aws_eks_addon" "cloudwatch" { +# cluster_name = aws_eks_cluster.cluster.name +# addon_name = "amazon-cloudwatch-observability" +# # addon_version = "v2.1.0-eksbuild.1" +# resolve_conflicts_on_create = "OVERWRITE" + +# pod_identity_association { +# role_arn = aws_iam_role.efs-addon.arn +# service_account = "efs-csi-controller-sa" +# } + +# configuration_values = jsonencode( +# { +# "controller" : { +# "nodeSelector" : { +# "karpenter.sh/nodepool" : "system" +# }, +# "tolerations" : [ +# { +# "key" : "CriticalAddonsOnly", +# "operator" : "Exists" +# }, +# { +# "effect" : "NoExecute", +# "operator" : "Exists", +# "tolerationSeconds" : 300 +# } +# ] +# } +# } +# ) +# } diff --git a/terraform/aws/aws-eks/module/eks.tf b/terraform/aws/aws-eks/module/eks.tf index aae15572..b5022967 100644 --- a/terraform/aws/aws-eks/module/eks.tf +++ b/terraform/aws/aws-eks/module/eks.tf @@ -50,8 +50,10 @@ resource "aws_eks_cluster" "cluster" { bootstrap_self_managed_addons = false compute_config { - enabled = true - node_pools = ["system", "general-purpose"] + enabled = true + # possible values "system", "general-purpose" + # deploying only system for system, addons deployment, nodepool for workflows is created via Helm module/cluster-config-chart + node_pools = ["system"] node_role_arn = aws_iam_role.node.arn } @@ -69,6 +71,9 @@ resource "aws_eks_cluster" "cluster" { } vpc_config { + # TODO make own instead, + # By default EKS Auto creates SecurityGroup allow access only from same SG and allowing all outbound + # security_group_ids = ... endpoint_private_access = true endpoint_public_access = true @@ -180,3 +185,19 @@ resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSNetworkingPolicy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy" role = aws_iam_role.cluster.name } + + +resource "null_resource" "cluster-config" { + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "${path.module}/configure-cluster.sh '${aws_eks_cluster.cluster.name}' '${var.region}' '${jsonencode(var.namespaces)}'" + } + + depends_on = [ + aws_iam_role.cluster, + aws_eks_access_entry.admin, + aws_eks_access_policy_association.admin, + ] +} diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/.helmignore b/terraform/aws/aws-eks/module/namespace-config-chart/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/Chart.yaml b/terraform/aws/aws-eks/module/namespace-config-chart/Chart.yaml new file mode 100644 index 00000000..d2039ecd --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: namespace-config +description: A Helm chart for EKS namespace configuration + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.31.0 diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/Makefile b/terraform/aws/aws-eks/module/namespace-config-chart/Makefile new file mode 100644 index 00000000..df8a8a7f --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/Makefile @@ -0,0 +1,10 @@ +deploy: ## deploy namespace configuration + helm upgrade --install namespace-config --namespace=cluster-config --create-namespace . + +undeploy: ## undeploys EKS configuration + helm uninstall namespace-config --namespace=cluster-config + +help: ## show usage and tasks (default) + @eval $$(sed -E -n 's/^([\*\.a-zA-Z0-9_-]+):.*?## (.*)$$/printf "\\033[36m%-30s\\033[0m %s\\n" "\1" "\2" ;/; ta; b; :a p' $(MAKEFILE_LIST)) +.DEFAULT_GOAL := help +.PHONY: deploy undeploy diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/templates/NOTES.txt b/terraform/aws/aws-eks/module/namespace-config-chart/templates/NOTES.txt new file mode 100644 index 00000000..5a051051 --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/templates/NOTES.txt @@ -0,0 +1 @@ +Install namespace configuration diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/templates/_helpers.tpl b/terraform/aws/aws-eks/module/namespace-config-chart/templates/_helpers.tpl new file mode 100644 index 00000000..ec33f77f --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "namespace-config.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "namespace-config.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "namespace-config.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "namespace-config.labels" -}} +helm.sh/chart: {{ include "namespace-config.chart" . }} +{{ include "namespace-config.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "namespace-config.selectorLabels" -}} +app.kubernetes.io/name: {{ include "namespace-config.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/templates/limitrange.yaml b/terraform/aws/aws-eks/module/namespace-config-chart/templates/limitrange.yaml new file mode 100644 index 00000000..24ce6825 --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/templates/limitrange.yaml @@ -0,0 +1,26 @@ +# yamllint disable-file +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limits + namespace: {{ .Values.namespace }} + labels: + {{- include "namespace-config.labels" . | nindent 4 }} +spec: + limits: + - type: "Container" + max: + cpu: "8" + memory: "8Gi" + min: + cpu: "10m" + memory: "4Mi" + default: + cpu: "300m" + memory: "200Mi" + defaultRequest: + cpu: "200m" + memory: "100Mi" + maxLimitRequestRatio: + cpu: "300" diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/templates/netpol.yaml b/terraform/aws/aws-eks/module/namespace-config-chart/templates/netpol.yaml new file mode 100644 index 00000000..e69de29b diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/templates/quota.yaml b/terraform/aws/aws-eks/module/namespace-config-chart/templates/quota.yaml new file mode 100644 index 00000000..da9dec25 --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/templates/quota.yaml @@ -0,0 +1,27 @@ +# yamllint disable-file +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: quota + namespace: {{ .Values.namespace }} + labels: + {{- include "namespace-config.labels" . | nindent 4 }} +spec: + hard: + configmaps: "100" + persistentvolumeclaims: "20" + requests.storage: "250Gi" + pods: "100" + replicationcontrollers: "20" + secrets: "50" + services: "20" + # TODO consider 0 to force developer to use Ingres/Route/VirtualService to expose service externally + services.loadbalancers: "5" + services.nodeports: "5" + requests.cpu: {{ .Values.quota.requests.cpu | default "16" | quote }} + requests.memory: {{ .Values.quota.requests.memory | default "32Gi" | quote }} + limits.cpu: {{ .Values.quota.limits.cpu | default "16" | quote }} + limits.memory: {{ .Values.quota.limits.memory | default "32Gi" | quote }} + requests.ephemeral-storage: "10Gi" + limits.ephemeral-storage: "10Gi" diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/templates/sa.yaml b/terraform/aws/aws-eks/module/namespace-config-chart/templates/sa.yaml new file mode 100644 index 00000000..46f12503 --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/templates/sa.yaml @@ -0,0 +1,30 @@ +# yamllint disable-file + +--- +{{- if .Values.dockerconfigjson }} +apiVersion: v1 +data: + .dockerconfigjson: {{ .Values.dockerconfigjson }} +kind: Secret +metadata: + name: {{ .Values.dockerreponame }} + namespace: {{ .Values.namespace }} + labels: + {{- include "namespace-config.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" # ensure to be installed before ResourceQuota, see quota.yaml for detais +type: kubernetes.io/dockerconfigjson +{{- end }} +--- +apiVersion: v1 +{{- if .Values.dockerconfigjson }} +imagePullSecrets: +- name: {{ .Values.dockerreponame }} +{{- end }} +kind: ServiceAccount +metadata: + name: app + namespace: {{ .Values.namespace }} + labels: + {{- include "namespace-config.labels" . | nindent 4 }} diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/values.yaml b/terraform/aws/aws-eks/module/namespace-config-chart/values.yaml new file mode 100644 index 00000000..c09f20ba --- /dev/null +++ b/terraform/aws/aws-eks/module/namespace-config-chart/values.yaml @@ -0,0 +1,10 @@ +namespace: namespace-name +# dockerconfigjson: base64encodedvalue +# dockerreponame: quay +quota: + requests: + cpu: "16" + memory: "32Gi" + limits: + cpu: "16" + memory: "32Gi" diff --git a/terraform/aws/aws-eks/module/pod-identity-associations.tf b/terraform/aws/aws-eks/module/pod-identity-associations.tf new file mode 100644 index 00000000..9b164049 --- /dev/null +++ b/terraform/aws/aws-eks/module/pod-identity-associations.tf @@ -0,0 +1,6 @@ +# resource "aws_eks_pod_identity_association" "association" { +# cluster_name = aws_eks_cluster.example.name +# namespace = var.namespace +# service_account = var.service_account_name +# role_arn = aws_iam_role.example.arn +# } diff --git a/terraform/aws/aws-eks/module/variables.tf b/terraform/aws/aws-eks/module/variables.tf index 1aa87077..f556643c 100644 --- a/terraform/aws/aws-eks/module/variables.tf +++ b/terraform/aws/aws-eks/module/variables.tf @@ -78,3 +78,36 @@ variable "cluster_admin_arn" { default = null description = "The break glass cluster-admin arn, if none provided, the arn of the creator is chosen" } + + + +variable "namespaces" { + type = list(object({ + name = string + quota = object({ + requests = object({ + cpu = string + memory = string + }) + limits = object({ + cpu = string + memory = string + }) + }) + })) + + description = "EKS namespaces configuration" + default = [{ + name = "test" + quota = { + limits = { + cpu = "12" + memory = "16Gi" + } + requests = { + cpu = "12" + memory = "16Gi" + } + } + }] +} diff --git a/terraform/aws/aws-eks/module/versions.tf b/terraform/aws/aws-eks/module/versions.tf index 591dcf95..158aaff3 100644 --- a/terraform/aws/aws-eks/module/versions.tf +++ b/terraform/aws/aws-eks/module/versions.tf @@ -4,6 +4,10 @@ terraform { source = "hashicorp/aws" version = "~> 5" } + null = { + source = "hashicorp/null" + version = "~> 3" + } } required_version = ">= 1.6" } diff --git a/terraform/aws/aws-eks/stage/dev/terragrunt.hcl b/terraform/aws/aws-eks/stage/dev/terragrunt.hcl index 0ca8d9ce..65f6c5d4 100644 --- a/terraform/aws/aws-eks/stage/dev/terragrunt.hcl +++ b/terraform/aws/aws-eks/stage/dev/terragrunt.hcl @@ -18,4 +18,18 @@ inputs = { Region = "us-east1" } zones = ["us-east-1a", "us-east-1b", "us-east-1c"] + + namespaces = [{ + name = "learning" + quota = { + limits = { + cpu = "12" + memory = "16Gi" + } + requests = { + cpu = "12" + memory = "16Gi" + } + } + }] } diff --git a/terraform/aws/aws-eks/test/deployment.yaml b/terraform/aws/aws-eks/test/deployment.yaml index 8c0e5619..4f0a0417 100644 --- a/terraform/aws/aws-eks/test/deployment.yaml +++ b/terraform/aws/aws-eks/test/deployment.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - namespace: game-2048 + namespace: learning name: deployment-2048 spec: selector: diff --git a/terraform/aws/aws-eks/test/ingress.yaml b/terraform/aws/aws-eks/test/ingress.yaml index 6cdc72aa..6fb9b509 100644 --- a/terraform/aws/aws-eks/test/ingress.yaml +++ b/terraform/aws/aws-eks/test/ingress.yaml @@ -1,13 +1,13 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - namespace: game-2048 + namespace: learning name: ingress-2048 annotations: alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/target-type: ip spec: - ingressClassName: alb + ingressClassName: alb-internal rules: - http: paths: diff --git a/terraform/aws/aws-eks/test/ingressclass.yaml b/terraform/aws/aws-eks/test/ingressclass.yaml deleted file mode 100644 index a7c94890..00000000 --- a/terraform/aws/aws-eks/test/ingressclass.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: eks.amazonaws.com/v1 -kind: IngressClassParams -metadata: - name: alb -spec: - scheme: internet-facing ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - name: alb - annotations: - # Use this annotation to set an IngressClass as Default - # If an Ingress doesn't specify a class, it will use the Default - ingressclass.kubernetes.io/is-default-class: "true" -spec: - # Configures the IngressClass to use EKS Auto Mode - controller: eks.amazonaws.com/alb - parameters: - apiGroup: eks.amazonaws.com - kind: IngressClassParams - # Use the name of the IngressClassParams set in the previous step - name: alb diff --git a/terraform/aws/aws-eks/test/service.yaml b/terraform/aws/aws-eks/test/service.yaml index 7c944ada..fdd059fc 100644 --- a/terraform/aws/aws-eks/test/service.yaml +++ b/terraform/aws/aws-eks/test/service.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - namespace: game-2048 + namespace: learning name: service-2048 spec: ports: