From 2903ee3896d2750493b58b43be43c26ebbd1928b Mon Sep 17 00:00:00 2001 From: Joey Davenport Date: Mon, 17 Oct 2022 13:19:38 -0600 Subject: [PATCH] Added more SG inputs;updated autoscaler perms;removed tiered storage submodule --- cluster_autoscaler.tf | 12 +- main.tf | 10 +- modules/tiered-storage-resources/README.md | 89 ------------- modules/tiered-storage-resources/main.tf | 119 ------------------ modules/tiered-storage-resources/outputs.tf | 38 ------ modules/tiered-storage-resources/variables.tf | 75 ----------- modules/tiered-storage-resources/versions.tf | 29 ----- variables.tf | 40 +++++- velero.tf | 8 +- 9 files changed, 61 insertions(+), 359 deletions(-) delete mode 100644 modules/tiered-storage-resources/README.md delete mode 100644 modules/tiered-storage-resources/main.tf delete mode 100644 modules/tiered-storage-resources/outputs.tf delete mode 100644 modules/tiered-storage-resources/variables.tf delete mode 100644 modules/tiered-storage-resources/versions.tf diff --git a/cluster_autoscaler.tf b/cluster_autoscaler.tf index 81d9b13..49f03e8 100644 --- a/cluster_autoscaler.tf +++ b/cluster_autoscaler.tf @@ -26,7 +26,11 @@ data "aws_iam_policy_document" "cluster_autoscaler" { "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", "autoscaling:DescribeTags", + "ec2:DescribeImages", + "ec2:DescribeInstanceTypes", "ec2:DescribeLaunchTemplateVersions", + "ec2:GetInstanceTypesFromInstanceRequirements", + "eks:DesribedNodegroup", ] resources = ["*"] @@ -62,10 +66,15 @@ data "aws_iam_policy_document" "cluster_autoscaler_sts" { identifiers = [format("arn:%s:iam::%s:oidc-provider/%s", local.aws_partition, local.account_id, local.oidc_issuer)] } condition { - test = "StringLike" + test = "StringEquals" values = [format("system:serviceaccount:%s:%s", "kube-system", "cluster-autoscaler")] variable = format("%s:sub", local.oidc_issuer) } + condition { + test = "StringEquals" + values = ["sts.amazonaws.com"] + variable = format("%s:aud", local.oidc_issuer) + } } } @@ -107,6 +116,7 @@ locals { "1.20" = "v1.20.1", "1.21" = "v1.21.1", "1.22" = "v1.22.1", + "1.23" = "v1.23.0" } } diff --git a/main.tf b/main.tf index 87aaae1..1d3a10b 100644 --- a/main.tf +++ b/main.tf @@ -150,13 +150,17 @@ module "eks" { aws_auth_roles = local.role_bindings cluster_name = var.cluster_name cluster_version = var.cluster_version - create_cluster_primary_security_group_tags = false - cluster_endpoint_private_access = true # Always set to true here, which enables private networking for the node groups + cluster_endpoint_private_access = true # Always set to true here, which enables private networking for the node groups cluster_endpoint_public_access = var.disable_public_eks_endpoint ? false : true cluster_endpoint_public_access_cidrs = var.allowed_public_cidrs cluster_enabled_log_types = var.cluster_enabled_log_types + cluster_security_group_additional_rules = var.cluster_security_group_additional_rules + cluster_security_group_id = var.cluster_security_group_id control_plane_subnet_ids = local.cluster_subnet_ids create_cloudwatch_log_group = false + create_cluster_primary_security_group_tags = false # Cleaner if we handle the tag in aws_ec2_tag.cluster_security_group + create_cluster_security_group = var.create_cluster_security_group + create_node_security_group = var.create_node_security_group create_iam_role = var.use_runtime_policy ? false : true eks_managed_node_groups = local.node_groups eks_managed_node_group_defaults = local.node_group_defaults @@ -165,6 +169,8 @@ module "eks" { iam_role_path = var.iam_path iam_role_permissions_boundary = var.permissions_boundary_arn manage_aws_auth_configmap = true + node_security_group_id = var.node_security_group_id + node_security_group_additional_rules = var.node_security_group_additional_rules openid_connect_audiences = ["sts.amazonaws.com"] tags = local.tags vpc_id = var.vpc_id diff --git a/modules/tiered-storage-resources/README.md b/modules/tiered-storage-resources/README.md deleted file mode 100644 index 6ea19dc..0000000 --- a/modules/tiered-storage-resources/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Tiered Storage for AWS -This Terraform module creates the resources needed for tiered storage offloading in Pulsar. This includes an encrypted and private S3 bucket and the IAM resources for IRSA. - -Here is an example usage: - -```hcl -module "tiered_storage" { - source = "streamnative/cloud/aws//modules/tiered_storage" - - cluster_name = "my-eks-cluster" - oidc_issuer = "oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E" - pulsar_namespace = "my-pulsar-namespace" - service_account_name = "my-pulsar-service-account" - - tags = { - Project = "MyApp" - Environment = "Prod" - } -} -``` - -## Important! -This module uses EKS IAM Roles for Service Accounts (IRSA). In order for these resources to work properly, there are two requirements: - -1. You must know the name of the Kubernetes Service Account and the Kubernetes namespace for your Pulsar workload. These don't need to exist prior to running this Terraform module, but are necessary for the resources to work properly. -2. You must add an annotation to the Service Account so it can be used by IRSA (IAM Role) created in this module. - -The module output includes the AWS ARN for the IAM role created. Using that, you can add the oppropriate annotation to the Service Account using `kubectl`: - -```shell -kubectl annotate serviceaccount -n \ -eks.amazonaws.com/role-arn=arn:aws:iam:::role/my-eks-cluster-tiered-storage-role -``` - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >=1.0.0 | -| [aws](#requirement\_aws) | >= 3.61.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.61.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_policy.tiered_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.tiered_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy_attachment.tiered_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_s3_bucket.tiered_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | -| [aws_s3_bucket_acl.tiered_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource | -| [aws_s3_bucket_server_side_encryption_configuration.tiered_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.tiered_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.tiered_storage_sts](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [aws\_partition](#input\_aws\_partition) | AWS partition: 'aws', 'aws-cn', or 'aws-us-gov' | `string` | `"aws"` | no | -| [cluster\_name](#input\_cluster\_name) | The name of your EKS cluster and associated resources | `string` | n/a | yes | -| [create\_iam\_policy\_for\_tiered\_storage](#input\_create\_iam\_policy\_for\_tiered\_storage) | Whether to create the IAM policy used by Pulsar's tiered storage offloading. For enhanced security, we allow for these IAM policies to be created seperately from this module. Defaults to "true". If set to "false", you must provide the ARN for the IAM policy needed for tiered storage offloading to function. | `bool` | `true` | no | -| [iam\_policy\_arn](#input\_iam\_policy\_arn) | The arn for the IAM policy used for Pulsar's tiered storage offloading. For enhanced security, we allow for IAM policies used by cluster addon services to be created seperately from this module. This is only required if the input "create\_iam\_policy\_for\_tiered\_storage" is set to "false". If created elsewhere, the expected name of the policy is "StreamNativeCloudTieredStoragedPolicy". | `string` | `null` | no | -| [kms\_key\_id](#input\_kms\_key\_id) | The KMS key ID to use for server side encryption. Defaults to "aws/s3". | `string` | `"aws/s3"` | no | -| [oidc\_issuer](#input\_oidc\_issuer) | The OIDC issuer for the EKS cluster | `string` | n/a | yes | -| [permissions\_boundary\_arn](#input\_permissions\_boundary\_arn) | If required, provide the ARN of the IAM permissions boundary to use for restricting StreamNative's vendor access. | `string` | `null` | no | -| [pulsar\_namespace](#input\_pulsar\_namespace) | The kubernetes namespace where Pulsar has been deployed. This is required to set the appropriate policy permissions for IRSA, which grants the Kubernetes Service Account access to use the IAM role | `string` | n/a | yes | -| [service\_account\_name](#input\_service\_account\_name) | The name of the kubernetes service account to by tiered storage offloading. Defaults to "pulsar". This is required to set the appropriate policy permissions for IRSA, which grants the Kubernetes Service Account access to use the IAM role | `string` | `"pulsar"` | no | -| [tags](#input\_tags) | Tags to be added to the bucket and corresponding resources | `map(string)` | `{}` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [role\_arn](#output\_role\_arn) | The arn of the role used for Pulsar's tiered storage offloading. This needs to be annotated on the corresponding Kubernetes Service account in order for IRSA to work properly, e.g. "eks.amazonaws.com/role-arn" : "" | -| [role\_name](#output\_role\_name) | The name of the role used for Pulsar's tiered storage offloading | -| [s3\_bucket](#output\_s3\_bucket) | The name of the bucket used for Pulsar's tiered storage offloading | -| [s3\_bucket\_arn](#output\_s3\_bucket\_arn) | The arn of the bucket used for Pulsar's tiered storage offloading | diff --git a/modules/tiered-storage-resources/main.tf b/modules/tiered-storage-resources/main.tf deleted file mode 100644 index 4d868bb..0000000 --- a/modules/tiered-storage-resources/main.tf +++ /dev/null @@ -1,119 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -data "aws_caller_identity" "current" {} - -data "aws_region" "current" {} - -resource "aws_s3_bucket" "tiered_storage" { - bucket = format("%s-storage-offload-%s", var.cluster_name, data.aws_region.current.name) - tags = merge({ "Vendor" = "StreamNative", "Attributes" = "offload", "Name" = "offload" }, var.tags) - - lifecycle { - ignore_changes = [ - bucket, - ] - } -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "tiered_storage" { - bucket = aws_s3_bucket.tiered_storage.bucket - rule { - apply_server_side_encryption_by_default { - kms_master_key_id = var.kms_key_id - sse_algorithm = "aws:kms" - } - } -} - -resource "aws_s3_bucket_acl" "tiered_storage" { - bucket = aws_s3_bucket.tiered_storage.id - acl = "private" -} - -data "aws_iam_policy_document" "tiered_storage" { - statement { - actions = [ - "s3:AbortMultipartUpload", - "s3:DeleteObject*", - "s3:GetObject*", - "s3:PutObject*", - "s3:List*", - ] - - resources = [ - aws_s3_bucket.tiered_storage.arn, - "${aws_s3_bucket.tiered_storage.arn}/*", - ] - } -} - -data "aws_iam_policy_document" "tiered_storage_sts" { - statement { - actions = [ - "sts:AssumeRoleWithWebIdentity" - ] - effect = "Allow" - principals { - type = "Federated" - identifiers = [format("arn:%s:iam::%s:oidc-provider/%s", var.aws_partition, data.aws_caller_identity.current.account_id, var.oidc_issuer)] - } - condition { - test = "StringLike" - values = [format("system:serviceaccount:%s:%s", var.pulsar_namespace, var.service_account_name)] - variable = format("%s:sub", var.oidc_issuer) - } - } -} - -resource "aws_iam_role" "tiered_storage" { - name = format("%s-tiered-storage-role", var.cluster_name) - description = format("Role used by IRSA and the KSA %s on StreamNative Cloud EKS cluster %s", var.cluster_name, var.service_account_name) - assume_role_policy = data.aws_iam_policy_document.tiered_storage_sts.json - path = "/StreamNative/" - permissions_boundary = var.permissions_boundary_arn - tags = merge({ "Vendor" = "StreamNative" }, var.tags) - - lifecycle { - ignore_changes = [ - assume_role_policy - ] - } -} - -resource "aws_iam_policy" "tiered_storage" { - count = var.create_iam_policy_for_tiered_storage ? 1 : 0 - name = format("%s-TieredStoragePolicy", var.cluster_name) - description = "Policy that defines the permissions for Pulsar's tiered storage offloading to S3, running in a StreamNative Cloud EKS cluster" - path = "/StreamNative/" - policy = data.aws_iam_policy_document.tiered_storage.json - tags = merge({ "Vendor" = "StreamNative" }, var.tags) - - lifecycle { - ignore_changes = [ - policy - ] - } -} - -resource "aws_iam_role_policy_attachment" "tiered_storage" { - count = var.create_iam_policy_for_tiered_storage ? 1 : 0 - policy_arn = var.create_iam_policy_for_tiered_storage ? aws_iam_policy.tiered_storage[0].arn : var.iam_policy_arn - role = aws_iam_role.tiered_storage.name -} \ No newline at end of file diff --git a/modules/tiered-storage-resources/outputs.tf b/modules/tiered-storage-resources/outputs.tf deleted file mode 100644 index abea3df..0000000 --- a/modules/tiered-storage-resources/outputs.tf +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -output "role_arn" { - value = aws_iam_role.tiered_storage.arn - description = "The arn of the role used for Pulsar's tiered storage offloading. This needs to be annotated on the corresponding Kubernetes Service account in order for IRSA to work properly, e.g. \"eks.amazonaws.com/role-arn\" : \"\"" -} - -output "role_name" { - value = aws_iam_role.tiered_storage.name - description = "The name of the role used for Pulsar's tiered storage offloading" -} - -output "s3_bucket" { - value = aws_s3_bucket.tiered_storage.bucket - description = "The name of the bucket used for Pulsar's tiered storage offloading" -} - -output "s3_bucket_arn" { - value = aws_s3_bucket.tiered_storage.arn - description = "The arn of the bucket used for Pulsar's tiered storage offloading" -} \ No newline at end of file diff --git a/modules/tiered-storage-resources/variables.tf b/modules/tiered-storage-resources/variables.tf deleted file mode 100644 index bc88b0a..0000000 --- a/modules/tiered-storage-resources/variables.tf +++ /dev/null @@ -1,75 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -variable "aws_partition" { - default = "aws" - description = "AWS partition: 'aws', 'aws-cn', or 'aws-us-gov'" - type = string -} - -variable "cluster_name" { - description = "The name of your EKS cluster and associated resources" - type = string -} - -variable "create_iam_policy_for_tiered_storage" { - default = true - description = "Whether to create the IAM policy used by Pulsar's tiered storage offloading. For enhanced security, we allow for these IAM policies to be created seperately from this module. Defaults to \"true\". If set to \"false\", you must provide the ARN for the IAM policy needed for tiered storage offloading to function." - type = bool -} - -variable "iam_policy_arn" { - default = null - description = "The arn for the IAM policy used for Pulsar's tiered storage offloading. For enhanced security, we allow for IAM policies used by cluster addon services to be created seperately from this module. This is only required if the input \"create_iam_policy_for_tiered_storage\" is set to \"false\". If created elsewhere, the expected name of the policy is \"StreamNativeCloudTieredStoragedPolicy\"." - type = string -} - -variable "kms_key_id" { - default = "aws/s3" - description = "The KMS key ID to use for server side encryption. Defaults to \"aws/s3\"." - type = string -} - -variable "oidc_issuer" { - description = "The OIDC issuer for the EKS cluster" - type = string -} - -variable "permissions_boundary_arn" { - default = null - description = "If required, provide the ARN of the IAM permissions boundary to use for restricting StreamNative's vendor access." - type = string -} - -variable "pulsar_namespace" { - description = "The kubernetes namespace where Pulsar has been deployed. This is required to set the appropriate policy permissions for IRSA, which grants the Kubernetes Service Account access to use the IAM role" - type = string -} - -variable "service_account_name" { - default = "pulsar" - description = "The name of the kubernetes service account to by tiered storage offloading. Defaults to \"pulsar\". This is required to set the appropriate policy permissions for IRSA, which grants the Kubernetes Service Account access to use the IAM role" - type = string -} - -variable "tags" { - default = {} - description = "Tags to be added to the bucket and corresponding resources" - type = map(string) -} diff --git a/modules/tiered-storage-resources/versions.tf b/modules/tiered-storage-resources/versions.tf deleted file mode 100644 index 8016768..0000000 --- a/modules/tiered-storage-resources/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -terraform { - required_version = ">=1.0.0" - - required_providers { - aws = { - version = ">= 4.0.0" - source = "hashicorp/aws" - } - } -} \ No newline at end of file diff --git a/variables.tf b/variables.tf index 7ae551d..3b45cbd 100644 --- a/variables.tf +++ b/variables.tf @@ -138,7 +138,7 @@ variable "cluster_autoscaler_helm_chart_repository" { } variable "cluster_autoscaler_helm_chart_version" { - default = "9.19.2" + default = "9.21.0" description = "Helm chart version for the cluster-autoscaler. Defaults to \"9.10.4\". See https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler for more details." type = string } @@ -166,6 +166,18 @@ variable "cluster_name" { } } +variable "cluster_security_group_additional_rules" { + default = {} + description = "Additional rules to add to the cluster security group. Set source_node_security_group = true inside rules to set the node_security_group as source." + type = any +} + +variable "cluster_security_group_id" { + default = "" + description = "The ID of an existing security group to use for the EKS cluster. If not provided, a new security group will be created." + type = string +} + variable "cluster_version" { default = "1.20" description = "The version of Kubernetes to be installed." @@ -196,12 +208,24 @@ variable "csi_settings" { type = map(any) } +variable "create_cluster_security_group" { + default = true + description = "Whether to create a new security group for the EKS cluster. If set to false, you must provide an existing security group via the cluster_security_group_id variable." + type = bool +} + variable "create_iam_policies" { default = true description = "Whether to create IAM policies for the IAM roles. If set to false, the module will default to using existing policy ARNs that must be present in the AWS account" type = bool } +variable "create_node_security_group" { + default = true + description = "Whether to create a new security group for the EKS nodes. If set to false, you must provide an existing security group via the node_security_group_id variable." + type = bool +} + variable "disable_public_eks_endpoint" { default = false description = "Whether to disable public access to the EKS control plane endpoint. If set to \"true\", additional configuration is required in order for the cluster to function properly, such as AWS PrivateLink for EC2, ECR, and S3, along with a VPN to access the EKS control plane. It is recommended to keep this setting to \"false\" unless you are familiar with this type of configuration." @@ -263,7 +287,7 @@ variable "external_dns_helm_chart_repository" { } variable "external_dns_helm_chart_version" { - default = "6.5.6" + default = "6.10.2" description = "Helm chart version for ExternalDNS. See https://hub.helm.sh/charts/bitnami/external-dns for updates." type = string } @@ -378,6 +402,18 @@ variable "metrics_server_settings" { type = map(any) } +variable "node_security_group_additional_rules" { + default = {} + description = "Additional ingress rules to add to the node security group. Set source_cluster_security_group = true inside rules to set the cluster_security_group as source" + type = any +} + +variable "node_security_group_id" { + default = "" + description = "An ID of an existing security group to use for the EKS node groups. If not specified, a new security group will be created." + type = string +} + variable "node_termination_handler_helm_chart_name" { default = "aws-node-termination-handler" description = "The name of the Helm chart to use for the AWS Node Termination Handler." diff --git a/velero.tf b/velero.tf index b491c77..5e51c41 100644 --- a/velero.tf +++ b/velero.tf @@ -139,12 +139,12 @@ resource "helm_release" "velero" { bucket = aws_s3_bucket.velero.id default = true config = { - region = var.region + region = var.region kmsKeyId = local.s3_kms_key } } volumeSnapshotLocation = { - name = "aws" + name = "aws" provider = "velero.io/aws" config = { region = var.region @@ -174,7 +174,7 @@ resource "helm_release" "velero" { serviceAccount = { server = { create = true - name = "velero" + name = "velero" annotations = { "eks.amazonaws.com/role-arn" = aws_iam_role.velero.arn } @@ -202,7 +202,7 @@ resource "helm_release" "velero" { } } - depends_on = [ + depends_on = [ kubernetes_namespace.velero ] }