From 3d3947a124d3ba7c76db35d5512d4b3a8371e75f Mon Sep 17 00:00:00 2001 From: LBH-wgreeff Date: Wed, 22 Nov 2023 15:54:14 +0000 Subject: [PATCH 1/2] Adding Redshift Serverless to Staging --- terraform/core/87-redshift-serverless.tf | 25 +++++ .../modules/redshift-serverless/00-init.tf | 15 +++ .../redshift-serverless/01-inputs-required.tf | 99 +++++++++++++++++++ .../redshift-serverless/02-inputs-optional.tf | 12 +++ .../redshift-serverless/03-inputs-derived.tf | 1 + .../modules/redshift-serverless/10-aws-iam.tf | 70 +++++++++++++ .../20-aws-redshiftserverless-namespace.tf | 20 ++++ .../21-aws-secrets-manager-secret.tf | 16 +++ .../redshift-serverless/22-aws-kms-key.tf | 31 ++++++ .../23-aws-redshiftserverless-workgroup.tf | 48 +++++++++ .../24-aws-security-group.tf | 30 ++++++ .../25-aws-redshiftserverless-usage-limit.tf | 7 ++ 12 files changed, 374 insertions(+) create mode 100644 terraform/core/87-redshift-serverless.tf create mode 100644 terraform/modules/redshift-serverless/00-init.tf create mode 100644 terraform/modules/redshift-serverless/01-inputs-required.tf create mode 100644 terraform/modules/redshift-serverless/02-inputs-optional.tf create mode 100644 terraform/modules/redshift-serverless/03-inputs-derived.tf create mode 100644 terraform/modules/redshift-serverless/10-aws-iam.tf create mode 100644 terraform/modules/redshift-serverless/20-aws-redshiftserverless-namespace.tf create mode 100644 terraform/modules/redshift-serverless/21-aws-secrets-manager-secret.tf create mode 100644 terraform/modules/redshift-serverless/22-aws-kms-key.tf create mode 100644 terraform/modules/redshift-serverless/23-aws-redshiftserverless-workgroup.tf create mode 100644 terraform/modules/redshift-serverless/24-aws-security-group.tf create mode 100644 terraform/modules/redshift-serverless/25-aws-redshiftserverless-usage-limit.tf diff --git a/terraform/core/87-redshift-serverless.tf b/terraform/core/87-redshift-serverless.tf new file mode 100644 index 000000000..c852dcda2 --- /dev/null +++ b/terraform/core/87-redshift-serverless.tf @@ -0,0 +1,25 @@ +module "redshift_serverless" { + count = local.is_live_environment && !local.is_production_environment ? 1 : 0 + source = "../modules/redshift-serverless" + tags = module.tags.values + subnet_ids = local.subnet_ids_list + identifier_prefix = local.identifier_prefix + secrets_manager_key = aws_kms_key.secrets_manager_key.id + vpc_id = data.aws_vpc.network.id + namespace_name = "${local.identifier_prefix}-redshift-serverless" + workgroup_name = "${local.identifier_prefix}-default" + admin_username = "data_engineers" + db_name = "data_platform" + workgroup_base_capacity = 16 + serverless_compute_usage_limit_period = "daily" + serverless_compute_usage_limit_amount = 1 + landing_zone_bucket_arn = module.landing_zone.bucket_arn + refined_zone_bucket_arn = module.refined_zone.bucket_arn + trusted_zone_bucket_arn = module.trusted_zone.bucket_arn + raw_zone_bucket_arn = module.raw_zone.bucket_arn + landing_zone_kms_key_arn = module.landing_zone.kms_key_arn + raw_zone_kms_key_arn = module.raw_zone.kms_key_arn + refined_zone_kms_key_arn = module.refined_zone.kms_key_arn + trusted_zone_kms_key_arn = module.trusted_zone.kms_key_arn +} + diff --git a/terraform/modules/redshift-serverless/00-init.tf b/terraform/modules/redshift-serverless/00-init.tf new file mode 100644 index 000000000..4aec9876f --- /dev/null +++ b/terraform/modules/redshift-serverless/00-init.tf @@ -0,0 +1,15 @@ +/* This defines the configuration of Terraform and AWS required Terraform Providers. + As this is a module, we don't have any explicity Provider blocks declared, as these + will be inherited from the parent Terraform. +*/ +terraform { + required_version = "~> 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + } +} + diff --git a/terraform/modules/redshift-serverless/01-inputs-required.tf b/terraform/modules/redshift-serverless/01-inputs-required.tf new file mode 100644 index 000000000..99a589fcc --- /dev/null +++ b/terraform/modules/redshift-serverless/01-inputs-required.tf @@ -0,0 +1,99 @@ +variable "tags" { + description = "AWS tags" + type = map(string) +} + +variable "identifier_prefix" { + description = "Prefix" + type = string +} + +variable "subnet_ids" { + description = "List of subnet ids" + type = list(string) +} + +variable "secrets_manager_key" { + description = "ARN of secrets manager KMS key" + type = string +} + +variable "vpc_id" { + description = "Id of vpc" + type = string +} + +variable "namespace_name" { + description = "Name of the namepsace to be created" + type = string +} + +variable "workgroup_name" { + description = "Name of the workgroup to be created" + type = string +} + +variable "admin_username" { + description = "Admin username for the workgroup" + type = string +} + +variable "db_name" { + description = "name of the database to be created" + type = string +} + +variable "serverless_compute_usage_limit_period" { + description = "Serverless compute usage limit period" + type = string + validation { + condition = contains(["daily", "weekly", "monthly"], var.serverless_compute_usage_limit_period) + error_message = "Invalid serverless_compute_usage_limit_period value" + } +} + +variable "serverless_compute_usage_limit_amount" { + description = "Usage limit amount (RPU)" + type = number +} + +variable "landing_zone_bucket_arn" { + description = "ARN of landing zone bucket" + type = string +} + +variable "refined_zone_bucket_arn" { + description = "ARN of refined zone bucket" + type = string +} + + +variable "trusted_zone_bucket_arn" { + description = "ARN of trusted zone bucket" + type = string +} + +variable "raw_zone_bucket_arn" { + description = "ARN of raw zone bucket" + type = string +} + +variable "landing_zone_kms_key_arn" { + description = "ARN of landing zone KMS key" + type = string +} + +variable "refined_zone_kms_key_arn" { + description = "ARN of refined zone KMS key" + type = string +} + +variable "trusted_zone_kms_key_arn" { + description = "ARN of trusted zone KMS key" + type = string +} + +variable "raw_zone_kms_key_arn" { + description = "ARN of raw zone KMS key" + type = string +} diff --git a/terraform/modules/redshift-serverless/02-inputs-optional.tf b/terraform/modules/redshift-serverless/02-inputs-optional.tf new file mode 100644 index 000000000..1d28db535 --- /dev/null +++ b/terraform/modules/redshift-serverless/02-inputs-optional.tf @@ -0,0 +1,12 @@ +variable "workgroup_base_capacity" { + description = "Base capacity of the workgroup in Redshift Processing Units (RPUs)" + type = number + default = 32 +} + + +# variable "maximimum_query_execution_time" { +# description = "Max query execution time (in seconds)" +# type = number +# default = 3600 +# } diff --git a/terraform/modules/redshift-serverless/03-inputs-derived.tf b/terraform/modules/redshift-serverless/03-inputs-derived.tf new file mode 100644 index 000000000..8fc4b38cc --- /dev/null +++ b/terraform/modules/redshift-serverless/03-inputs-derived.tf @@ -0,0 +1 @@ +data "aws_caller_identity" "current" {} diff --git a/terraform/modules/redshift-serverless/10-aws-iam.tf b/terraform/modules/redshift-serverless/10-aws-iam.tf new file mode 100644 index 000000000..3841ed64a --- /dev/null +++ b/terraform/modules/redshift-serverless/10-aws-iam.tf @@ -0,0 +1,70 @@ +data "aws_iam_policy_document" "redshift_serverless_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + identifiers = ["redshift.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_iam_role" "redshift_serverless_role" { + tags = var.tags + + name = "${var.identifier_prefix}-redshift-serverless-role" + assume_role_policy = data.aws_iam_policy_document.redshift_serverless_role.json +} + +data "aws_iam_policy_document" "redshift_serverless" { + statement { + actions = [ + "s3:ListBucket", + "s3:GetObject" + ] + #should this have access to landing zone? + resources = [ + "${var.landing_zone_bucket_arn}/*", + var.landing_zone_bucket_arn, + "${var.refined_zone_bucket_arn}/*", + var.refined_zone_bucket_arn, + "${var.trusted_zone_bucket_arn}/*", + var.trusted_zone_bucket_arn, + "${var.raw_zone_bucket_arn}/*", + var.raw_zone_bucket_arn + ] + } + + statement { + actions = [ + "glue:*" + ] + + resources = [ + "*", + ] + } + + statement { + actions = [ + "kms:Decrypt", + ] + resources = [ + var.landing_zone_kms_key_arn, + var.raw_zone_kms_key_arn, + var.refined_zone_kms_key_arn, + var.trusted_zone_kms_key_arn, + ] + } +} + +resource "aws_iam_policy" "redshift_serverless_access_policy" { + name = "${var.identifier_prefix}-redshift-serverless-access-policy" + policy = data.aws_iam_policy_document.redshift_serverless.json +} + +resource "aws_iam_role_policy_attachment" "redshift_serverless_role_policy_attachment" { + role = aws_iam_role.redshift_serverless_role.name + policy_arn = aws_iam_policy.redshift_serverless_access_policy.arn +} + diff --git a/terraform/modules/redshift-serverless/20-aws-redshiftserverless-namespace.tf b/terraform/modules/redshift-serverless/20-aws-redshiftserverless-namespace.tf new file mode 100644 index 000000000..61c7f59cc --- /dev/null +++ b/terraform/modules/redshift-serverless/20-aws-redshiftserverless-namespace.tf @@ -0,0 +1,20 @@ +resource "aws_redshiftserverless_namespace" "namespace" { + tags = var.tags + + namespace_name = var.namespace_name + + admin_user_password = aws_secretsmanager_secret_version.master_password.secret_string + admin_username = var.admin_username + db_name = var.db_name + default_iam_role_arn = aws_iam_role.redshift_serverless_role.arn + iam_roles = [aws_iam_role.redshift_serverless_role.arn] + kms_key_id = aws_kms_key.key.arn + + # #this is not ideal and can cause headaches if roles need tweaking, but seems to be a known issue https://github.com/hashicorp/terraform-provider-aws/issues/26624 + # lifecycle { + # ignore_changes = [ + # iam_roles + # ] + # } +} + diff --git a/terraform/modules/redshift-serverless/21-aws-secrets-manager-secret.tf b/terraform/modules/redshift-serverless/21-aws-secrets-manager-secret.tf new file mode 100644 index 000000000..dddef59a5 --- /dev/null +++ b/terraform/modules/redshift-serverless/21-aws-secrets-manager-secret.tf @@ -0,0 +1,16 @@ +resource "random_password" "master_password" { + length = 40 + special = false +} + +resource "aws_secretsmanager_secret" "master_password" { + name_prefix = "${var.identifier_prefix}-redshift-serverless-${var.namespace_name}-namespace-master-password" + description = "Master password for redshift serverless ${var.namespace_name} namespace" + kms_key_id = var.secrets_manager_key +} + +resource "aws_secretsmanager_secret_version" "master_password" { + secret_id = aws_secretsmanager_secret.master_password.id + secret_string = random_password.master_password.result +} + diff --git a/terraform/modules/redshift-serverless/22-aws-kms-key.tf b/terraform/modules/redshift-serverless/22-aws-kms-key.tf new file mode 100644 index 000000000..0652da4c9 --- /dev/null +++ b/terraform/modules/redshift-serverless/22-aws-kms-key.tf @@ -0,0 +1,31 @@ +resource "aws_kms_key" "key" { + tags = var.tags + + description = "${var.identifier_prefix}-redshift-serverless-${var.namespace_name}-namespace" + deletion_window_in_days = 10 + enable_key_rotation = true + + policy = data.aws_iam_policy_document.key_policy.json +} + +data "aws_iam_policy_document" "key_policy" { + statement { + effect = "Allow" + actions = [ + "kms:*" + ] + + resources = ["*"] + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + } + + } +} + +resource "aws_kms_alias" "name" { + name = lower("alias/${var.identifier_prefix}-redshift-serverless-${var.namespace_name}-namespace") + target_key_id = aws_kms_key.key.id +} diff --git a/terraform/modules/redshift-serverless/23-aws-redshiftserverless-workgroup.tf b/terraform/modules/redshift-serverless/23-aws-redshiftserverless-workgroup.tf new file mode 100644 index 000000000..7b315bc65 --- /dev/null +++ b/terraform/modules/redshift-serverless/23-aws-redshiftserverless-workgroup.tf @@ -0,0 +1,48 @@ +resource "aws_redshiftserverless_workgroup" "default" { + tags = var.tags + + namespace_name = aws_redshiftserverless_namespace.namespace.namespace_name + workgroup_name = var.workgroup_name + base_capacity = var.workgroup_base_capacity + #config_parameter + enhanced_vpc_routing = false + publicly_accessible = false + security_group_ids = [aws_security_group.redshift_serverless.id] + subnet_ids = var.subnet_ids + + # #setting one will set all existing ones to null, also these can't be updated in a single call. Would have to apply the values one by one + # config_parameter { + # parameter_key = "max_query_execution_time" + # parameter_value = var.maximimum_query_execution_time + # } + + # # config_parameter { + # # parameter_key = "auto_mv" + # # parameter_value = true + # # } + # config_parameter { + # parameter_key = "datestyle" + # parameter_value = "ISO, MDY" + # } + # # config_parameter { + # # parameter_key = "enable_case_sensitive_identifier" + # # parameter_value = "false" + # # } + # config_parameter { + # parameter_key = "enable_user_activity_logging" + # parameter_value = "true" + # } + # config_parameter { + # parameter_key = "query_group" + # parameter_value = "default" + # } + # config_parameter { + # parameter_key = "search_path" + # parameter_value = "$user, public" + # } + # config_parameter { + # parameter_key = "auto_mv" + # parameter_value = "true" + # } +} + diff --git a/terraform/modules/redshift-serverless/24-aws-security-group.tf b/terraform/modules/redshift-serverless/24-aws-security-group.tf new file mode 100644 index 000000000..0c37f7d40 --- /dev/null +++ b/terraform/modules/redshift-serverless/24-aws-security-group.tf @@ -0,0 +1,30 @@ +resource "aws_security_group" "redshift_serverless" { + tags = var.tags + + name = "${var.identifier_prefix}-redshift-serverless-${var.namespace_name}-namespace" + description = "Restrict access to redshift serverless" + vpc_id = var.vpc_id + revoke_rules_on_delete = true +} + +#TODO: lock these down +resource "aws_security_group_rule" "redshift_serverless_ingress" { + description = "Allow all inbound traffic" + type = "ingress" + from_port = 0 + to_port = 0 + protocol = "TCP" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.redshift_serverless.id +} + +resource "aws_security_group_rule" "redshift_serverless_egress" { + description = "Allows all outbound traffic" + type = "egress" + from_port = 0 + to_port = 0 + protocol = "TCP" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.redshift_serverless.id +} + diff --git a/terraform/modules/redshift-serverless/25-aws-redshiftserverless-usage-limit.tf b/terraform/modules/redshift-serverless/25-aws-redshiftserverless-usage-limit.tf new file mode 100644 index 000000000..ef0ddeeaf --- /dev/null +++ b/terraform/modules/redshift-serverless/25-aws-redshiftserverless-usage-limit.tf @@ -0,0 +1,7 @@ +resource "aws_redshiftserverless_usage_limit" "usage_limit" { + resource_arn = aws_redshiftserverless_workgroup.default.arn + usage_type = "serverless-compute" + period = var.serverless_compute_usage_limit_period + amount = var.serverless_compute_usage_limit_amount + breach_action = "log" +} From 349b4e49ac09bb233bd6f26c558c8349f275eb06 Mon Sep 17 00:00:00 2001 From: LBH-wgreeff Date: Wed, 22 Nov 2023 16:39:33 +0000 Subject: [PATCH 2/2] updating workgroup base capacity --- terraform/core/87-redshift-serverless.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/core/87-redshift-serverless.tf b/terraform/core/87-redshift-serverless.tf index c852dcda2..c6f6d0141 100644 --- a/terraform/core/87-redshift-serverless.tf +++ b/terraform/core/87-redshift-serverless.tf @@ -10,7 +10,7 @@ module "redshift_serverless" { workgroup_name = "${local.identifier_prefix}-default" admin_username = "data_engineers" db_name = "data_platform" - workgroup_base_capacity = 16 + workgroup_base_capacity = 32 serverless_compute_usage_limit_period = "daily" serverless_compute_usage_limit_amount = 1 landing_zone_bucket_arn = module.landing_zone.bucket_arn