From 11a674fa3952da334e78623ae003feab7521c5ed Mon Sep 17 00:00:00 2001 From: fredericfran-gds Date: Mon, 15 Aug 2022 21:14:12 +0100 Subject: [PATCH] Add AWS EFS CSI driver and create Clamav EFS We want to create a EFS volume to store the Clamav virus database. Before, we attempted to create the EFS in AWS only and mount it as NFS in the Clamav pod but this was unsuccessful since the EFS has root ownership and the pod is running as non-root. We could create the access point of the EFS and sets the same ownership as the pod in AWS but we would have to use the volume handler option of k8s PersistentVolume for k8s to mount the access point properly. This option requires the AWS EFS CSI driver as a prerequisite. The chosen solution is: 1. install AWS EFS CSI driver and associated permissions 2. The storage class requires a EFS id so we create the EFS and pass it to the driver. --- .../cluster-infrastructure/aws_ebs_csi_iam.tf | 8 +-- .../cluster-infrastructure/aws_efs_csi_iam.tf | 66 +++++++++++++++++++ .../cluster-infrastructure/clamav_db_efs.tf | 31 +++++++++ .../cluster-infrastructure/outputs.tf | 18 ++++- .../cluster-services/aws_ebs_csi_driver.tf | 2 +- .../cluster-services/aws_efs_csi_driver.tf | 31 +++++++++ 6 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf create mode 100644 terraform/deployments/cluster-infrastructure/clamav_db_efs.tf create mode 100644 terraform/deployments/cluster-services/aws_efs_csi_driver.tf diff --git a/terraform/deployments/cluster-infrastructure/aws_ebs_csi_iam.tf b/terraform/deployments/cluster-infrastructure/aws_ebs_csi_iam.tf index bd948de80..a5586d4d9 100644 --- a/terraform/deployments/cluster-infrastructure/aws_ebs_csi_iam.tf +++ b/terraform/deployments/cluster-infrastructure/aws_ebs_csi_iam.tf @@ -1,16 +1,16 @@ locals { - csi_driver_controller_service_account_name = "ebs-csi-controller-sa" + ebs_csi_driver_controller_service_account_name = "ebs-csi-controller-sa" } module "aws_ebs_csi_driver_iam_role" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" version = "~> 4.0" create_role = true - role_name = "${local.csi_driver_controller_service_account_name}-${var.cluster_name}" - role_description = "Role for the AWS EBS CSI driver controller. Corresponds to ${local.csi_driver_controller_service_account_name} k8s ServiceAccount." + role_name = "${local.ebs_csi_driver_controller_service_account_name}-${var.cluster_name}" + role_description = "Role for the AWS EBS CSI driver controller. Corresponds to ${local.ebs_csi_driver_controller_service_account_name} k8s ServiceAccount." provider_url = module.eks.oidc_provider role_policy_arns = [aws_iam_policy.aws_ebs_csi_driver.arn] - oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:${local.csi_driver_controller_service_account_name}"] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:${local.ebs_csi_driver_controller_service_account_name}"] } resource "aws_iam_policy" "aws_ebs_csi_driver" { diff --git a/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf b/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf new file mode 100644 index 000000000..102eb382d --- /dev/null +++ b/terraform/deployments/cluster-infrastructure/aws_efs_csi_iam.tf @@ -0,0 +1,66 @@ +locals { + efs_csi_driver_controller_service_account_name = "efs-csi-controller-sa" +} + +module "aws_efs_csi_driver_iam_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "~> 4.0" + create_role = true + role_name = "${local.efs_csi_driver_controller_service_account_name}-${var.cluster_name}" + role_description = "Role for the AWS EFS CSI driver controller. Corresponds to ${local.efs_csi_driver_controller_service_account_name} k8s ServiceAccount." + provider_url = module.eks.oidc_provider + role_policy_arns = [aws_iam_policy.aws_efs_csi_driver.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:${local.efs_csi_driver_controller_service_account_name}"] +} + +resource "aws_iam_role_policy_attachment" "eks_nodes_efs" { + role = module.eks.eks_managed_node_groups["main"].iam_role_name + policy_arn = aws_iam_policy.aws_efs_csi_driver.arn +} + +resource "aws_iam_policy" "aws_efs_csi_driver" { + name = "AWSEfsCsiController-${var.cluster_name}" + description = "Allow the driver to manage AWS EFS" + + # The argument to jsonencode() is the verbatim contents of + # https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/iam-policy-example.json + # (except for whitespace changes from terraform fmt). + policy = jsonencode({ + + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeMountTargets", + "ec2:DescribeAvailabilityZones" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticfilesystem:CreateAccessPoint" + ], + "Resource" : "*", + "Condition" : { + "StringLike" : { + "aws:RequestTag/efs.csi.aws.com/cluster" : "true" + } + } + }, + { + "Effect" : "Allow", + "Action" : "elasticfilesystem:DeleteAccessPoint", + "Resource" : "*", + "Condition" : { + "StringEquals" : { + "aws:ResourceTag/efs.csi.aws.com/cluster" : "true" + } + } + } + ] + }) +} diff --git a/terraform/deployments/cluster-infrastructure/clamav_db_efs.tf b/terraform/deployments/cluster-infrastructure/clamav_db_efs.tf new file mode 100644 index 000000000..e0cb83997 --- /dev/null +++ b/terraform/deployments/cluster-infrastructure/clamav_db_efs.tf @@ -0,0 +1,31 @@ +locals { + clamav_db_name = "clamav-db-${var.cluster_name}" +} + +resource "aws_efs_file_system" "clamav-db" { + creation_token = local.clamav_db_name + tags = { "Description" = "EFS where Clamav virus signature database is stored" } +} + +resource "aws_security_group" "clamav-db" { + name = local.clamav_db_name + vpc_id = data.terraform_remote_state.infra_vpc.outputs.vpc_id + description = "Security group of ${local.clamav_db_name}" +} + +resource "aws_security_group_rule" "clamav_db_from_eks_workers" { + description = "Clamav DB EFS accepts requests from EKS nodes" + type = "ingress" + from_port = 2049 + to_port = 2049 + protocol = "tcp" + security_group_id = aws_security_group.clamav-db.id + source_security_group_id = local.node_security_group_id +} + +resource "aws_efs_mount_target" "clamav-db-mount-targets" { + for_each = toset(data.terraform_remote_state.infra_networking.outputs.private_subnet_ids) + file_system_id = aws_efs_file_system.clamav-db.id + subnet_id = each.key + security_groups = [aws_security_group.clamav-db.id] +} diff --git a/terraform/deployments/cluster-infrastructure/outputs.tf b/terraform/deployments/cluster-infrastructure/outputs.tf index 4d0d78fa3..813bec10e 100644 --- a/terraform/deployments/cluster-infrastructure/outputs.tf +++ b/terraform/deployments/cluster-infrastructure/outputs.tf @@ -18,6 +18,11 @@ output "aws_ebs_csi_driver_iam_role_arn" { value = module.aws_ebs_csi_driver_iam_role.iam_role_arn } +output "aws_efs_csi_driver_iam_role_arn" { + description = "IAM role ARN for AWS EFS CSI controller role" + value = module.aws_efs_csi_driver_iam_role.iam_role_arn +} + output "control_plane_security_group_id" { description = "ID of the security group which contains the (AWS-owned) control plane nodes." value = module.eks.cluster_primary_security_group_id @@ -94,8 +99,13 @@ output "aws_lb_controller_service_account_name" { } output "aws_ebs_csi_driver_controller_service_account_name" { - description = "Name of the k8s service account for the AWS Ebs Csi Controller" - value = local.csi_driver_controller_service_account_name + description = "Name of the k8s service account for the AWS EBS CSI Controller" + value = local.ebs_csi_driver_controller_service_account_name +} + +output "aws_efs_csi_driver_controller_service_account_name" { + description = "Name of the k8s service account for the AWS EFS CSI Controller" + value = local.efs_csi_driver_controller_service_account_name } output "grafana_iam_role_arn" { @@ -107,3 +117,7 @@ output "monitoring_namespace" { description = "The namespace for monitoring." value = local.monitoring_namespace } + +output "clamav_db_efs_id" { + value = aws_efs_file_system.clamav-db.id +} diff --git a/terraform/deployments/cluster-services/aws_ebs_csi_driver.tf b/terraform/deployments/cluster-services/aws_ebs_csi_driver.tf index 33c0fb9f7..ffaa64fb1 100644 --- a/terraform/deployments/cluster-services/aws_ebs_csi_driver.tf +++ b/terraform/deployments/cluster-services/aws_ebs_csi_driver.tf @@ -1,4 +1,4 @@ -resource "helm_release" "csi_driver" { +resource "helm_release" "ebs_csi_driver" { chart = "aws-ebs-csi-driver" name = "aws-ebs-csi-driver" namespace = "kube-system" diff --git a/terraform/deployments/cluster-services/aws_efs_csi_driver.tf b/terraform/deployments/cluster-services/aws_efs_csi_driver.tf new file mode 100644 index 000000000..d7ca9e6ff --- /dev/null +++ b/terraform/deployments/cluster-services/aws_efs_csi_driver.tf @@ -0,0 +1,31 @@ +resource "helm_release" "efs_csi_driver" { + chart = "aws-efs-csi-driver" + name = "aws-efs-csi-driver" + namespace = "kube-system" + repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver" + version = "2.2.7" # TODO: Dependabot or equivalent so this doesn't get neglected. + + values = [yamlencode({ + controller = { + serviceAccount = { + create = true + name = data.terraform_remote_state.cluster_infrastructure.outputs.aws_efs_csi_driver_controller_service_account_name + annotations = { + "eks.amazonaws.com/role-arn" = data.terraform_remote_state.cluster_infrastructure.outputs.aws_efs_csi_driver_iam_role_arn + } + } + } + storageClasses = [{ + name = "clamav-db-efs-sc" + apiVersion = "storage.k8s.io/v1" + mountOptions = ["tls"] + parameters = { + provisioningMode = "efs-ap" + fileSystemId = data.terraform_remote_state.cluster_infrastructure.outputs.clamav_db_efs_id + directoryPerms = "755" + } + reclaimPolicy = "Retain" + volumeBindingMode = "WaitForFirstConsumer" + }] + })] +}