Adding Redshift Serverless to Staging

terraform/core/87-redshift-serverless.tf

@@ -0,0 +1,25 @@
+module "redshift_serverless" {
+  count                                 = local.is_live_environment && !local.is_production_environment ? 1 : 0
+  source                                = "../modules/redshift-serverless"
+  tags                                  = module.tags.values
+  subnet_ids                            = local.subnet_ids_list
+  identifier_prefix                     = local.identifier_prefix
+  secrets_manager_key                   = aws_kms_key.secrets_manager_key.id
+  vpc_id                                = data.aws_vpc.network.id
+  namespace_name                        = "${local.identifier_prefix}-redshift-serverless"
+  workgroup_name                        = "${local.identifier_prefix}-default"
+  admin_username                        = "data_engineers"
+  db_name                               = "data_platform"
+  workgroup_base_capacity               = 16
+  serverless_compute_usage_limit_period = "daily"
+  serverless_compute_usage_limit_amount = 1
+  landing_zone_bucket_arn               = module.landing_zone.bucket_arn
+  refined_zone_bucket_arn               = module.refined_zone.bucket_arn
+  trusted_zone_bucket_arn               = module.trusted_zone.bucket_arn
+  raw_zone_bucket_arn                   = module.raw_zone.bucket_arn
+  landing_zone_kms_key_arn              = module.landing_zone.kms_key_arn
+  raw_zone_kms_key_arn                  = module.raw_zone.kms_key_arn
+  refined_zone_kms_key_arn              = module.refined_zone.kms_key_arn
+  trusted_zone_kms_key_arn              = module.trusted_zone.kms_key_arn
+}
+

terraform/modules/redshift-serverless/00-init.tf

@@ -0,0 +1,15 @@
+/* This defines the configuration of Terraform and AWS required Terraform Providers.
+   As this is a module, we don't have any explicity Provider blocks declared, as these
+   will be inherited from the parent Terraform.
+*/
+terraform {
+  required_version = "~> 1.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+

terraform/modules/redshift-serverless/01-inputs-required.tf

@@ -0,0 +1,99 @@
+variable "tags" {
+  description = "AWS tags"
+  type        = map(string)
+}
+
+variable "identifier_prefix" {
+  description = "Prefix"
+  type        = string
+}
+
+variable "subnet_ids" {
+  description = "List of subnet ids"
+  type        = list(string)
+}
+
+variable "secrets_manager_key" {
+  description = "ARN of secrets manager KMS key"
+  type        = string
+}
+
+variable "vpc_id" {
+  description = "Id of vpc"
+  type        = string
+}
+
+variable "namespace_name" {
+  description = "Name of the namepsace to be created"
+  type        = string
+}
+
+variable "workgroup_name" {
+  description = "Name of the workgroup to be created"
+  type        = string
+}
+
+variable "admin_username" {
+  description = "Admin username for the workgroup"
+  type        = string
+}
+
+variable "db_name" {
+  description = "name of the database to be created"
+  type        = string
+}
+
+variable "serverless_compute_usage_limit_period" {
+  description = "Serverless compute usage limit period"
+  type        = string
+  validation {
+    condition     = contains(["daily", "weekly", "monthly"], var.serverless_compute_usage_limit_period)
+    error_message = "Invalid serverless_compute_usage_limit_period value"
+  }
+}
+
+variable "serverless_compute_usage_limit_amount" {
+  description = "Usage limit amount (RPU)"
+  type        = number
+}
+
+variable "landing_zone_bucket_arn" {
+  description = "ARN of landing zone bucket"
+  type        = string
+}
+
+variable "refined_zone_bucket_arn" {
+  description = "ARN of refined zone bucket"
+  type        = string
+}
+
+
+variable "trusted_zone_bucket_arn" {
+  description = "ARN of trusted zone bucket"
+  type        = string
+}
+
+variable "raw_zone_bucket_arn" {
+  description = "ARN of raw zone bucket"
+  type        = string
+}
+
+variable "landing_zone_kms_key_arn" {
+  description = "ARN of landing zone KMS key"
+  type        = string
+}
+
+variable "refined_zone_kms_key_arn" {
+  description = "ARN of refined zone KMS key"
+  type        = string
+}
+
+variable "trusted_zone_kms_key_arn" {
+  description = "ARN of trusted zone KMS key"
+  type        = string
+}
+
+variable "raw_zone_kms_key_arn" {
+  description = "ARN of raw zone KMS key"
+  type        = string
+}

terraform/modules/redshift-serverless/02-inputs-optional.tf

@@ -0,0 +1,12 @@
+variable "workgroup_base_capacity" {
+  description = "Base capacity of the workgroup in Redshift Processing Units (RPUs)"
+  type        = number
+  default     = 32
+}
+
+
+# variable "maximimum_query_execution_time" {
+#   description = "Max query execution time (in seconds)"
+#   type        = number
+#   default     = 3600
+# }

terraform/modules/redshift-serverless/03-inputs-derived.tf

@@ -0,0 +1 @@
+data "aws_caller_identity" "current" {}

terraform/modules/redshift-serverless/10-aws-iam.tf

@@ -0,0 +1,70 @@
+data "aws_iam_policy_document" "redshift_serverless_role" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      identifiers = ["redshift.amazonaws.com"]
+      type        = "Service"
+    }
+  }
+}
+
+resource "aws_iam_role" "redshift_serverless_role" {
+  tags = var.tags
+
+  name               = "${var.identifier_prefix}-redshift-serverless-role"
+  assume_role_policy = data.aws_iam_policy_document.redshift_serverless_role.json
+}
+
+data "aws_iam_policy_document" "redshift_serverless" {
+  statement {
+    actions = [
+      "s3:ListBucket",
+      "s3:GetObject"
+    ]
+    #should this have access to landing zone?
+    resources = [
+      "${var.landing_zone_bucket_arn}/*",
+      var.landing_zone_bucket_arn,
+      "${var.refined_zone_bucket_arn}/*",
+      var.refined_zone_bucket_arn,
+      "${var.trusted_zone_bucket_arn}/*",
+      var.trusted_zone_bucket_arn,
+      "${var.raw_zone_bucket_arn}/*",
+      var.raw_zone_bucket_arn
+    ]
+  }
+
+  statement {
+    actions = [
+      "glue:*"
+    ]
+
+    resources = [
+      "*",
+    ]
+  }
+
+  statement {
+    actions = [
+      "kms:Decrypt",
+    ]
+    resources = [
+      var.landing_zone_kms_key_arn,
+      var.raw_zone_kms_key_arn,
+      var.refined_zone_kms_key_arn,
+      var.trusted_zone_kms_key_arn,
+    ]
+  }
+}
+
+resource "aws_iam_policy" "redshift_serverless_access_policy" {
+  name   = "${var.identifier_prefix}-redshift-serverless-access-policy"
+  policy = data.aws_iam_policy_document.redshift_serverless.json
+}
+
+resource "aws_iam_role_policy_attachment" "redshift_serverless_role_policy_attachment" {
+  role       = aws_iam_role.redshift_serverless_role.name
+  policy_arn = aws_iam_policy.redshift_serverless_access_policy.arn
+}
+

terraform/modules/redshift-serverless/20-aws-redshiftserverless-namespace.tf

@@ -0,0 +1,20 @@
+resource "aws_redshiftserverless_namespace" "namespace" {
+  tags = var.tags
+
+  namespace_name = var.namespace_name
+
+  admin_user_password  = aws_secretsmanager_secret_version.master_password.secret_string
+  admin_username       = var.admin_username
+  db_name              = var.db_name
+  default_iam_role_arn = aws_iam_role.redshift_serverless_role.arn
+  iam_roles            = [aws_iam_role.redshift_serverless_role.arn]
+  kms_key_id           = aws_kms_key.key.arn
+
+  # #this is not ideal and can cause headaches if roles need tweaking, but seems to be a known issue https://github.com/hashicorp/terraform-provider-aws/issues/26624
+  # lifecycle {
+  #   ignore_changes = [
+  #     iam_roles
+  #   ]
+  # }
+}
+

terraform/modules/redshift-serverless/21-aws-secrets-manager-secret.tf

@@ -0,0 +1,16 @@
+resource "random_password" "master_password" {
+  length  = 40
+  special = false
+}
+
+resource "aws_secretsmanager_secret" "master_password" {
+  name_prefix = "${var.identifier_prefix}-redshift-serverless-${var.namespace_name}-namespace-master-password"
+  description = "Master password for redshift serverless ${var.namespace_name} namespace"
+  kms_key_id  = var.secrets_manager_key
+}
+
+resource "aws_secretsmanager_secret_version" "master_password" {
+  secret_id     = aws_secretsmanager_secret.master_password.id
+  secret_string = random_password.master_password.result
+}
+

terraform/modules/redshift-serverless/22-aws-kms-key.tf

@@ -0,0 +1,31 @@
+resource "aws_kms_key" "key" {
+  tags = var.tags
+
+  description             = "${var.identifier_prefix}-redshift-serverless-${var.namespace_name}-namespace"
+  deletion_window_in_days = 10
+  enable_key_rotation     = true
+
+  policy = data.aws_iam_policy_document.key_policy.json
+}
+
+data "aws_iam_policy_document" "key_policy" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:*"
+    ]
+
+    resources = ["*"]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+
+  }
+}
+
+resource "aws_kms_alias" "name" {
+  name          = lower("alias/${var.identifier_prefix}-redshift-serverless-${var.namespace_name}-namespace")
+  target_key_id = aws_kms_key.key.id
+}

terraform/modules/redshift-serverless/23-aws-redshiftserverless-workgroup.tf

@@ -0,0 +1,48 @@
+resource "aws_redshiftserverless_workgroup" "default" {
+  tags = var.tags
+
+  namespace_name = aws_redshiftserverless_namespace.namespace.namespace_name
+  workgroup_name = var.workgroup_name
+  base_capacity  = var.workgroup_base_capacity
+  #config_parameter
+  enhanced_vpc_routing = false
+  publicly_accessible  = false
+  security_group_ids   = [aws_security_group.redshift_serverless.id]
+  subnet_ids           = var.subnet_ids
+
+  # #setting one will set all existing ones to null, also these can't be updated in a single call. Would have to apply the values one by one
+  # config_parameter {
+  #     parameter_key = "max_query_execution_time"
+  #     parameter_value  = var.maximimum_query_execution_time
+  # }
+
+  # #  config_parameter {
+  # #       parameter_key   = "auto_mv"
+  # #       parameter_value = true
+  # #     }
+  #  config_parameter {
+  #      parameter_key   = "datestyle"
+  #      parameter_value = "ISO, MDY"
+  #     }
+  # #  config_parameter {
+  # #      parameter_key   = "enable_case_sensitive_identifier"
+  # #      parameter_value = "false"
+  # #     }
+  #  config_parameter {
+  #      parameter_key   = "enable_user_activity_logging"
+  #      parameter_value = "true"
+  #     }
+  #  config_parameter {
+  #      parameter_key   = "query_group"
+  #      parameter_value = "default"
+  #     }
+  #  config_parameter {
+  #      parameter_key   = "search_path"
+  #      parameter_value = "$user, public"
+  #     }
+  # config_parameter {
+  #             parameter_key = "auto_mv"
+  #             parameter_value = "true"
+  #           }
+}
+

terraform/modules/redshift-serverless/24-aws-security-group.tf

@@ -0,0 +1,30 @@
+resource "aws_security_group" "redshift_serverless" {
+  tags = var.tags
+
+  name                   = "${var.identifier_prefix}-redshift-serverless-${var.namespace_name}-namespace"
+  description            = "Restrict access to redshift serverless"
+  vpc_id                 = var.vpc_id
+  revoke_rules_on_delete = true
+}
+
+#TODO: lock these down
+resource "aws_security_group_rule" "redshift_serverless_ingress" {
+  description       = "Allow all inbound traffic"
+  type              = "ingress"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "TCP"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = aws_security_group.redshift_serverless.id
+}
+
+resource "aws_security_group_rule" "redshift_serverless_egress" {
+  description       = "Allows all outbound traffic"
+  type              = "egress"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "TCP"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = aws_security_group.redshift_serverless.id
+}
+

terraform/modules/redshift-serverless/25-aws-redshiftserverless-usage-limit.tf

@@ -0,0 +1,7 @@
+resource "aws_redshiftserverless_usage_limit" "usage_limit" {
+  resource_arn  = aws_redshiftserverless_workgroup.default.arn
+  usage_type    = "serverless-compute"
+  period        = var.serverless_compute_usage_limit_period
+  amount        = var.serverless_compute_usage_limit_amount
+  breach_action = "log"
+}