From f05c4ab7c421601c8079cceb33ba864ee457040d Mon Sep 17 00:00:00 2001 From: Mateusz Nowakowski Date: Wed, 18 Dec 2024 22:27:36 +0100 Subject: [PATCH] feat: add pod identity, irsa, cloud watch addon to EKS deployment --- k8s/apps/echoserver/scripts/deploy-on-eks.sh | 4 +- terraform/aws/aws-eks/Makefile | 21 ++++++- terraform/aws/aws-eks/README.md | 19 ++++-- .../aws/aws-eks/module/configure-cluster.sh | 15 ++++- terraform/aws/aws-eks/module/eks-addons.tf | 58 ++++++++----------- terraform/aws/aws-eks/module/eks.tf | 33 ++++++++++- terraform/aws/aws-eks/module/irsa.tf | 51 ++++++++++++++++ .../namespace-config-chart/templates/sa.yaml | 16 +++++ .../module/namespace-config-chart/values.yaml | 3 + .../module/pod-identity-associations.tf | 42 ++++++++++++-- terraform/aws/aws-eks/module/variables.tf | 10 ++-- terraform/aws/aws-eks/module/versions.tf | 4 ++ .../aws/aws-eks/stage/dev/terragrunt.hcl | 17 +++--- terraform/aws/aws-iam-linked/module/roles.tf | 10 ++++ 14 files changed, 239 insertions(+), 64 deletions(-) create mode 100644 terraform/aws/aws-eks/module/irsa.tf diff --git a/k8s/apps/echoserver/scripts/deploy-on-eks.sh b/k8s/apps/echoserver/scripts/deploy-on-eks.sh index b61dfadc..642a851a 100644 --- a/k8s/apps/echoserver/scripts/deploy-on-eks.sh +++ b/k8s/apps/echoserver/scripts/deploy-on-eks.sh @@ -13,8 +13,8 @@ kubectl config use-context minikube || echo "Minikube not present in kube contex --set ingress.tls.crt="$(base64 -w 0 /tmp/${CN}.crt)" \ --set ingress.tls.key="$(base64 -w 0 /tmp/${CN}.key)" \ --set ingress.host="${CN}" \ - --set ingress.class=nginx \ - --set networkPolicy.enabled=true + --set ingress.class=external-alb \ + --set networkPolicy.enabled=false # while [ -z "$(kubectl get ingress echoserver -n learning -o jsonpath="{.status..ip}" | xargs)" ]; do # sleep 1 diff --git a/terraform/aws/aws-eks/Makefile b/terraform/aws/aws-eks/Makefile index 918c4b5b..4be4350e 100644 --- a/terraform/aws/aws-eks/Makefile +++ b/terraform/aws/aws-eks/Makefile @@ -29,7 +29,7 @@ run: init ## setup VPC: make run [ENV=dev] [MODE=apply] kubeconfig: ## generate kubeconfig entry for EKS cluster aws eks --region $(shell cd stage/$(ENV) && terragrunt output -raw region) update-kubeconfig --name $(shell cd stage/$(ENV) && terragrunt output -raw cluster_name) -kubeconfig-oidc: ## test OIDC setup and provide instruction to work +oidc-test: ## test OIDC setup and provide instructions to update ~/.kube/config to use it ifndef ISSUER_URL $(error Environment ISSUER_URL is not defined. For keycloak it would some some: https://id.yourdomain.com/realms/NAME_OF_REALM) endif @@ -44,6 +44,25 @@ endif --oidc-client-id=$(CLIENT_ID) \ --oidc-client-secret=$(CLIENT_SECRET) +kubeconfig-oidc: kubeconfig ## create ~/.kube/config entry for EKS access (using public API endpoint) with user authentication via OIDC +ifndef ISSUER_URL + $(error Environment ISSUER_URL is not defined. For keycloak it would some some: https://id.yourdomain.com/realms/NAME_OF_REALM) +endif +ifndef CLIENT_ID + $(error Environment CLIENT_ID is not defined. For keycloak it would be realm oidc Client ID, for example: eks) +endif +ifndef CLIENT_SECRET + $(error Environment CLIENT_SECRET is not defined. For keycloak it would be realm oidc Client ID credentials secret) +endif + kubectl config set-credentials oidc \ + --exec-api-version=client.authentication.k8s.io/v1beta1 \ + --exec-command=kubectl \ + --exec-arg=oidc-login \ + --exec-arg=get-token \ + --exec-arg=--oidc-issuer-url=$(ISSUER_URL) \ + --exec-arg=--oidc-client-id=$(CLIENT_ID) \ + --exec-arg=--oidc-client-secret=$(CLIENT_SECRET) + kubectl config set-context --current --user=oidc show-state: ## show state cd stage/$(ENV) && terragrunt state list && terragrunt show diff --git a/terraform/aws/aws-eks/README.md b/terraform/aws/aws-eks/README.md index c84b7e74..f1cca2cb 100644 --- a/terraform/aws/aws-eks/README.md +++ b/terraform/aws/aws-eks/README.md @@ -4,13 +4,17 @@ Terraform scripts deploy EKS In particular it creates: -- EKS with Auto mode -- EFS and CSI Snapshots addons +- EKS with Auto mode - with two NodePools: + - `system` - buildin for critical workflows + - custom `compute-spot-arm64` - for as cheap as possible workflows based on ARM64 and Spot instances +- Cloud Watch, EFS and CSI Snapshots addons - Configure storage, ingress, node pool classes -- Configure app namespaces (quuota, limits, networkpolicies) +- Configure app namespaces (quota, limits, networkpolicies) + - For each managed namespace creates `edit` and `view` rolebinding for Groups `NSNAME-edit` and `NSNAME-view` + - Assign optionally `EKS Pod Identity Role` to namespace `app` Service Account. + - Assign optionally `IRSA` to namespace `app-irsa` Service Account. - Configure OIDC authen & authz (Keycloak tested) with Group - Add ClusterRoleBinding allowing `cluster-admin` for Group: `cluster-admins` - - For each managed namespace creates `edit` and `view` rolebinding for Groups `NSNAME-edit` and `NSNAME-view` ## Prerequisites @@ -63,11 +67,14 @@ make show-state ENV=dev make kubeconfig -# test OIDC integration and provide instruction how to make kubeconfig using oidc +# test OIDC setup and provide instructions to update ~/.kube/config to use it export CLIENT_SECRET=...clientid_secret_from_keycloak.... -make oidc-setup-test ISSUER_URL=https://id.yoursite.com/realms/yourrealm CLIENT_ID=eks +make oidc-test ISSUER_URL=https://id.yoursite.com/realms/yourrealm CLIENT_ID=eks +# when all is ok, JWT token contains valid claims (especially groups) then +# create ~/.kube/config entry for EKS access (using public API endpoint) with user authentication via OIDC +make kubeconfig-oidc ISSUER_URL=https://id.yoursite.com/realms/yourrealm CLIENT_ID=eks ``` ## Day 2 Operations diff --git a/terraform/aws/aws-eks/module/configure-cluster.sh b/terraform/aws/aws-eks/module/configure-cluster.sh index 3aba3d6c..5a760f65 100755 --- a/terraform/aws/aws-eks/module/configure-cluster.sh +++ b/terraform/aws/aws-eks/module/configure-cluster.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash -CLUSTER_NAME="${1:?CLUSTER_NAME is required}" -REGION="${2:?REGION is required}" -NAMESPACES="${3:?NAMESPACES is required}" +ACCOUNT_ID="${1:?CLUSTER_NAME is required}" +CLUSTER_NAME="${2:?CLUSTER_NAME is required}" +REGION="${3:?REGION is required}" +NAMESPACES="${4:?NAMESPACES is required}" set -e set -x @@ -21,10 +22,18 @@ for NAMESPACE in $(echo "${NAMESPACES}" | jq -cr '.[]'); do NS="$(echo "${NAMESPACE}" | jq -r ".name")" QUOTA="$(echo "${NAMESPACE}" | jq -r ".quota")" + IRSA_ROLE="" + IRSA_POLICY="$(echo "${NAMESPACE}" | jq -r ".irsa_policy")" + [[ -n "${IRSA_POLICY}" && "${IRSA_POLICY}" != "null" ]] && { + IRSA_ROLE="${CLUSTER_NAME}-${NS}-irsa" + } + [ -n "$(kubectl get ns "${NS}" --no-headers --ignore-not-found)" ] || { kubectl create ns "${NS}" } helm upgrade --install "ns-${NS}-config" -n cluster-config --create-namespace "${DIRNAME}/namespace-config-chart" \ --set namespace="${NS}" \ + --set aws.accountId="${ACCOUNT_ID}" \ + --set irsaRole="${IRSA_ROLE}" \ --set-json quota="$(echo "${QUOTA}" | jq -r)" done diff --git a/terraform/aws/aws-eks/module/eks-addons.tf b/terraform/aws/aws-eks/module/eks-addons.tf index 6a0df331..4cdd6ff1 100644 --- a/terraform/aws/aws-eks/module/eks-addons.tf +++ b/terraform/aws/aws-eks/module/eks-addons.tf @@ -81,38 +81,30 @@ resource "aws_iam_role_policy_attachment" "efs-addon_AmazonEFSCSIDriverPolicy" { role = aws_iam_role.efs-addon.name } -# TODO CloudWatch addons does not supoport Pod Identifies yet, only IRSA - -# # https://docs.aws.amazon.com/eks/latest/userguide/workloads-add-ons-available-eks.html#add-ons-aws-efs-csi-driver -# resource "aws_eks_addon" "cloudwatch" { -# cluster_name = aws_eks_cluster.cluster.name -# addon_name = "amazon-cloudwatch-observability" -# # addon_version = "v2.1.0-eksbuild.1" -# resolve_conflicts_on_create = "OVERWRITE" +# CloudWatch addons does not support Pod Identifies yet, only IRSA or node level role +# TODO evaluate more configuration options +# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/install-CloudWatch-Observability-EKS-addon.html +# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/metrics-collected-by-CloudWatch-agent.html +resource "aws_eks_addon" "cloudwatch" { + cluster_name = aws_eks_cluster.cluster.name + addon_name = "amazon-cloudwatch-observability" + # addon_version = "v2.1.0-eksbuild.1" + resolve_conflicts_on_create = "OVERWRITE" -# pod_identity_association { -# role_arn = aws_iam_role.efs-addon.arn -# service_account = "efs-csi-controller-sa" -# } -# configuration_values = jsonencode( -# { -# "controller" : { -# "nodeSelector" : { -# "karpenter.sh/nodepool" : "system" -# }, -# "tolerations" : [ -# { -# "key" : "CriticalAddonsOnly", -# "operator" : "Exists" -# }, -# { -# "effect" : "NoExecute", -# "operator" : "Exists", -# "tolerationSeconds" : 300 -# } -# ] -# } -# } -# ) -# } + configuration_values = jsonencode( + { "containerLogs" : { "enabled" : true }, + "tolerations" : [ + { + "key" : "CriticalAddonsOnly", + "operator" : "Exists" + }, + { + "effect" : "NoExecute", + "operator" : "Exists", + "tolerationSeconds" : 300 + } + ] + } + ) +} diff --git a/terraform/aws/aws-eks/module/eks.tf b/terraform/aws/aws-eks/module/eks.tf index b5022967..0d28de13 100644 --- a/terraform/aws/aws-eks/module/eks.tf +++ b/terraform/aws/aws-eks/module/eks.tf @@ -132,16 +132,47 @@ resource "aws_iam_role" "node" { }) } + +# To collect AmazonEBS Volume IDs in Logs via EKS CloudWatch Addon +# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/install-CloudWatch-Observability-EKS-addon.html +resource "aws_iam_role_policy" "node_ReadEBSVolumeIDs" { + name = "CollectEBSvolumeIDs" + role = aws_iam_role.node.id + + policy = < namespace if namespace.irsa_policy != null } + + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.oidc_provider.arn] + } + condition { + test = "StringEquals" + variable = "${aws_iam_openid_connect_provider.oidc_provider.url}:sub" + values = ["system:serviceaccount:${each.key}:app-irsa"] + } + } +} + + +resource "aws_iam_role" "irsa" { + for_each = { for namespace in var.namespaces : namespace.name => namespace if namespace.irsa_policy != null } + + name = "${local.prefix}-${each.key}-irsa" + assume_role_policy = data.aws_iam_policy_document.irsa_trust_policy[each.key].json +} + +# TODO make irsa_policy a list of policies to assing to a role +resource "aws_iam_role_policy_attachment" "irsa" { + for_each = { for namespace in var.namespaces : namespace.name => namespace if namespace.irsa_policy != null } + + policy_arn = each.value.irsa_policy + role = aws_iam_role.irsa[each.key].name +} diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/templates/sa.yaml b/terraform/aws/aws-eks/module/namespace-config-chart/templates/sa.yaml index 46f12503..e14c7d23 100644 --- a/terraform/aws/aws-eks/module/namespace-config-chart/templates/sa.yaml +++ b/terraform/aws/aws-eks/module/namespace-config-chart/templates/sa.yaml @@ -28,3 +28,19 @@ metadata: namespace: {{ .Values.namespace }} labels: {{- include "namespace-config.labels" . | nindent 4 }} +{{- if .Values.irsaRole }} +--- +apiVersion: v1 +{{- if .Values.dockerconfigjson }} +imagePullSecrets: +- name: {{ .Values.dockerreponame }} +{{- end }} +kind: ServiceAccount +metadata: + name: app-irsa + namespace: {{ .Values.namespace }} + labels: + {{- include "namespace-config.labels" . | nindent 4 }} + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.aws.accountId }}:role/{{ .Values.irsaRole}} +{{- end }} diff --git a/terraform/aws/aws-eks/module/namespace-config-chart/values.yaml b/terraform/aws/aws-eks/module/namespace-config-chart/values.yaml index c09f20ba..6fb7f98a 100644 --- a/terraform/aws/aws-eks/module/namespace-config-chart/values.yaml +++ b/terraform/aws/aws-eks/module/namespace-config-chart/values.yaml @@ -1,6 +1,9 @@ namespace: namespace-name # dockerconfigjson: base64encodedvalue # dockerreponame: quay +aws: + accountId: "" +# irsaRole: "..." quota: requests: cpu: "16" diff --git a/terraform/aws/aws-eks/module/pod-identity-associations.tf b/terraform/aws/aws-eks/module/pod-identity-associations.tf index 9b164049..be495f7e 100644 --- a/terraform/aws/aws-eks/module/pod-identity-associations.tf +++ b/terraform/aws/aws-eks/module/pod-identity-associations.tf @@ -1,6 +1,38 @@ -# resource "aws_eks_pod_identity_association" "association" { -# cluster_name = aws_eks_cluster.example.name -# namespace = var.namespace -# service_account = var.service_account_name -# role_arn = aws_iam_role.example.arn +resource "aws_eks_pod_identity_association" "association" { + for_each = { for namespace in var.namespaces : namespace.name => namespace if namespace.pod_identity_role != null } + + cluster_name = aws_eks_cluster.cluster.name + namespace = each.key + service_account = "app" + role_arn = data.aws_iam_role.association[each.key].arn +} + + +# EKS Pod Identity roles has to have the following trust policy: +# Example: +# resource "aws_iam_role" "s3all" { +# name = "s3all" + +# assume_role_policy = <<-EOF +# { +# "Version": "2012-10-17", +# "Statement": [ +# { +# "Effect": "Allow", +# "Principal": { +# "Service": "pods.eks.amazonaws.com" +# }, +# "Action": [ +# "sts:AssumeRole", +# "sts:TagSession" +# ] +# } +# ] +# } +# EOF # } +data "aws_iam_role" "association" { + for_each = { for namespace in var.namespaces : namespace.name => namespace if namespace.pod_identity_role != null } + + name = each.value.pod_identity_role +} diff --git a/terraform/aws/aws-eks/module/variables.tf b/terraform/aws/aws-eks/module/variables.tf index 8113aba5..5ef509c6 100644 --- a/terraform/aws/aws-eks/module/variables.tf +++ b/terraform/aws/aws-eks/module/variables.tf @@ -10,7 +10,6 @@ data "aws_iam_session_context" "current" { locals { - # tflint-ignore: terraform_unused_declarations account_id = data.aws_caller_identity.current.account_id prefix = "${var.name != "" ? "${var.name}-" : ""}${var.env}-${var.region}" } @@ -83,8 +82,10 @@ variable "cluster_admin_arn" { variable "namespaces" { type = list(object({ - name = string - fargate = bool + name = string + pod_identity_role = optional(string) + irsa_policy = optional(string) + fargate = optional(bool, false) quota = object({ requests = object({ cpu = string @@ -99,8 +100,7 @@ variable "namespaces" { description = "EKS namespaces configuration" default = [{ - name = "test" - fargate = false + name = "test" quota = { limits = { cpu = "12" diff --git a/terraform/aws/aws-eks/module/versions.tf b/terraform/aws/aws-eks/module/versions.tf index 158aaff3..2b262352 100644 --- a/terraform/aws/aws-eks/module/versions.tf +++ b/terraform/aws/aws-eks/module/versions.tf @@ -8,6 +8,10 @@ terraform { source = "hashicorp/null" version = "~> 3" } + tls = { + source = "hashicorp/tls" + version = "~> 4" + } } required_version = ">= 1.6" } diff --git a/terraform/aws/aws-eks/stage/dev/terragrunt.hcl b/terraform/aws/aws-eks/stage/dev/terragrunt.hcl index 2a27a74f..8592c080 100644 --- a/terraform/aws/aws-eks/stage/dev/terragrunt.hcl +++ b/terraform/aws/aws-eks/stage/dev/terragrunt.hcl @@ -20,16 +20,17 @@ inputs = { zones = ["us-east-1a", "us-east-1b", "us-east-1c"] # Uncomment to integrated cluster authen/authz with OIDC - # oidc = { - # issuer_url = "https://id.yourdomain.com/realms/yourrealm" - # client_id = "eks" - # username_claim = "email" - # groups_claim = "groups" - # } + oidc = { + issuer_url = "https://id.matihost.mooo.com/realms/id" + client_id = "eks" + username_claim = "email" + groups_claim = "groups" + } namespaces = [{ - name = "learning" - fargate = false + name = "learning" + pod_identity_role = "s3all" + irsa_policy = "arn:aws:iam::aws:policy/AmazonS3FullAccess" quota = { limits = { cpu = "12" diff --git a/terraform/aws/aws-iam-linked/module/roles.tf b/terraform/aws/aws-iam-linked/module/roles.tf index 95a2f3d8..4dd54f1a 100644 --- a/terraform/aws/aws-iam-linked/module/roles.tf +++ b/terraform/aws/aws-iam-linked/module/roles.tf @@ -12,6 +12,16 @@ resource "aws_iam_role" "s3all" { "Service": "ec2.amazonaws.com" }, "Action": "sts:AssumeRole" + }, + { + "Effect": "Allow", + "Principal": { + "Service": "pods.eks.amazonaws.com" + }, + "Action": [ + "sts:AssumeRole", + "sts:TagSession" + ] } ] }