feat: add nightly testing eks (#250)

## Description
Add EKS setup for deploying and testing uds-core.

## Related Issue
related to #213 

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [ ] Test, docs, adr added or updated as needed
- [ ] [Contributor Guide
Steps](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)(https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md#submitting-a-pull-request)
followed

---------

Co-authored-by: zamaz <[email protected]>

.github/bundles/uds-bundle.yaml

@@ -0,0 +1,50 @@
+kind: UDSBundle
+metadata:
+  name: uds-core-eks-nightly
+  description: A UDS bundle for deploying EKS and UDS Core
+  # x-release-please-start-version
+  version: "0.18.0"
+  # x-release-please-end
+
+packages:
+  - name: init
+    repository: ghcr.io/defenseunicorns/packages/init
+    # renovate: datasource=github-tags depName=defenseunicorns/zarf versioning=semver
+    ref: v0.32.4
+
+  - name: core
+    path: ../../build/
+    # x-release-please-start-version
+    ref: 0.18.0
+    # x-release-please-end
+    overrides:
+      velero:
+        velero:
+          variables:
+            - name: VELERO_USE_SECRET
+              description: "Toggle use secret off to use IRSA."
+              path: credentials.useSecret
+            - name: VELERO_IRSA_ANNOTATION
+              description: "IRSA ARN annotation to use for Velero"
+              path: serviceAccount.server.annotations
+      loki:
+        loki:
+          variables:
+            - name: LOKI_CHUNKS_BUCKET
+              description: "The object storage bucket for Loki chunks"
+              path: loki.storage.bucketNames.chunks
+            - name: LOKI_RULER_BUCKET
+              description: "The object storage bucket for Loki ruler"
+              path: loki.storage.bucketNames.ruler
+            - name: LOKI_ADMIN_BUCKET
+              description: "The object storage bucket for Loki admin"
+              path: loki.storage.bucketNames.admin
+            - name: LOKI_S3_ENDPOINT
+              description: "The S3 endpoint"
+              path: loki.storage.s3.endpoint
+            - name: LOKI_S3_REGION
+              description: "The S3 region"
+              path: loki.storage.s3.region
+            - name: LOKI_IRSA_ANNOTATION
+              description: "The irsa role annotation"
+              path: loki.serviceAccount.annotations

.github/bundles/uds-config.yaml

@@ -0,0 +1,21 @@
+# Overwritten by ci-iac-aws package
+options:
+  architecture: amd64
+
+variables:
+  core:
+    loki_chunks_bucket: ${ZARF_VAR_LOKI_S3_BUCKET}
+    loki_ruler_bucket: ${ZARF_VAR_LOKI_S3_BUCKET}
+    loki_admin_bucket: ${ZARF_VAR_LOKI_S3_BUCKET}
+    loki_s3_region: ${ZARF_VAR_LOKI_S3_AWS_REGION}
+    loki_s3-endpoint: ""
+    loki_irsa_annotation:
+      eks.amazonaws.com/role-arn: ${ZARF_VAR_LOKI_S3_ROLE_ARN}
+    velero_use_secret: false
+    velero_irsa_annotation:
+      eks.amazonaws.com/role-arn: "${ZARF_VAR_VELERO_S3_ROLE_ARN}"
+    velero_bucket: ${ZARF_VAR_VELERO_S3_BUCKET}
+    velero_bucket_region: ${ZARF_VAR_VELERO_S3_AWS_REGION}
+    velero_bucket_provider_url: ""
+    velero_bucket_credential_name: ""
+    velero_bucket_credential_key: ""

.github/test-infra/buckets-iac/loki.tf

@@ -0,0 +1,28 @@
+resource "aws_iam_policy" "loki_policy" {
+  name        = "${local.bucket_configurations.loki.name}-irsa-${random_id.unique_id.hex}"
+  path        = "/"
+  description = "IAM policy for Loki to have necessary permissions to use S3 for storing logs."
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["s3:ListBucket"]
+        Resource = ["arn:${data.aws_partition.current.partition}:s3:::${module.S3["loki"].bucket_name}"]
+      },
+      {
+        Effect   = "Allow"
+        Action   = ["s3:*Object"]
+        Resource = ["arn:${data.aws_partition.current.partition}:s3:::${module.S3["loki"].bucket_name}/*"]
+      },
+      {
+        Effect = "Allow"
+        Action = [
+          "kms:GenerateDataKey",
+          "kms:Decrypt"
+        ]
+        Resource = [local.kms_key_arns["loki"].kms_key_arn]
+      }
+    ]
+  })
+}

.github/test-infra/buckets-iac/main.tf

@@ -0,0 +1,130 @@
+provider "aws" {
+  region = var.region
+
+  default_tags {
+    tags = {
+      PermissionsBoundary = var.permissions_boundary_name
+    }
+  }
+}
+
+terraform {
+  required_version = "1.5.7"
+  backend "s3" {
+  }
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 4.0, != 5.17.0"
+    }
+
+    random = {
+      source  = "hashicorp/random"
+      version = "3.5.1"
+    }
+  }
+}
+
+resource "random_id" "default" {
+  byte_length = 2
+}
+
+data "aws_eks_cluster" "existing" {
+  name = var.name
+}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_partition" "current" {}
+
+data "aws_region" "current" {}
+
+locals {
+  oidc_url_without_protocol = substr(data.aws_eks_cluster.existing.identity[0].oidc[0].issuer, 8, -1)
+  oidc_arn                  = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_url_without_protocol}"
+  iam_role_permissions_boundary = var.use_permissions_boundary ? "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/${var.permissions_boundary_name}" : null
+
+  bucket_configurations = {
+    for instance in var.bucket_configurations :
+    instance.name => {
+      name = "${var.name}-${instance.name}"
+      service_account = instance.service_account
+      namespace = instance.namespace
+    } 
+  }
+
+  kms_key_arns = module.generate_kms
+
+  iam_policies = {
+    "loki" = resource.aws_iam_policy.loki_policy.arn
+    "velero" = resource.aws_iam_policy.velero_policy.arn
+  }
+}
+
+resource "random_id" "unique_id" {
+  byte_length = 4
+}
+
+module "generate_kms" {
+  for_each = local.bucket_configurations
+  source = "github.com/defenseunicorns/terraform-aws-uds-kms?ref=v0.0.2"
+
+  key_owners = var.key_owner_arns
+  # A list of IAM ARNs for those who will have full key permissions (`kms:*`)
+  kms_key_alias_name_prefix = "${each.value.name}-" # Prefix for KMS key alias.
+  kms_key_deletion_window   = var.kms_key_deletion_window
+  # Waiting period for scheduled KMS Key deletion. Can be 7-30 days.
+  kms_key_description = "${var.name} UDS Core deployment Loki Key" # Description for the KMS key.
+  tags = {
+    Deployment = "UDS Core ${each.value.name}"
+  }
+}
+
+module "S3" {
+  for_each = local.bucket_configurations
+  source                  = "github.com/defenseunicorns/terraform-aws-uds-s3?ref=v0.0.6"
+  name_prefix             = "${each.value.name}-"
+  kms_key_arn             = local.kms_key_arns[each.key].kms_key_arn
+  force_destroy           = "true"
+  create_bucket_lifecycle = true
+}
+
+module "irsa" {
+  for_each = local.bucket_configurations
+  source                        = "github.com/defenseunicorns/terraform-aws-uds-irsa?ref=v0.0.2"
+  name                          = each.value.name
+  kubernetes_service_account    = each.value.service_account
+  kubernetes_namespace          = each.value.namespace
+  oidc_provider_arn             = local.oidc_arn
+  role_permissions_boundary_arn = local.iam_role_permissions_boundary
+
+  role_policy_arns = tomap({
+    "${each.key}" = local.iam_policies[each.key]
+  })
+}
+
+resource "aws_s3_bucket_policy" "bucket_policy" {
+  for_each = local.bucket_configurations
+  bucket = module.S3[each.key].bucket_name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "s3:ListBucket",
+          "s3:GetObject",
+          "s3:PutObject"
+        ]
+        Effect = "Allow"
+        Principal = {
+          AWS = module.irsa[each.key].role_arn
+        }
+        Resource = [
+          module.S3[each.key].bucket_arn,
+          "${module.S3[each.key].bucket_arn}/*"
+        ]
+      }
+    ]
+  })
+}

.github/test-infra/buckets-iac/output.tf

@@ -0,0 +1,27 @@
+output "aws_region" {
+  value = data.aws_region.current.name
+}
+
+output "loki_irsa_role_arn" {
+  value = module.irsa["loki"].role_arn
+}
+
+output "loki_s3" {
+  value = module.S3["loki"]
+}
+
+output "loki_s3_bucket" {
+  value = module.S3["loki"].bucket_name
+}
+
+output "velero_irsa_role_arn" {
+  value = module.irsa["velero"].role_arn
+}
+
+output "velero_s3" {
+  value = module.S3["velero"]
+}
+
+output "velero_s3_bucket" {
+  value = module.S3["velero"].bucket_name
+}

.github/test-infra/buckets-iac/variables.tf

@@ -0,0 +1,55 @@
+variable "region" {
+  description = "AWS region"
+  type        = string
+}
+
+variable "name" {
+  description = "Name for cluster"
+  type        = string
+}
+
+variable "permissions_boundary_name" {
+  description = "The name of the permissions boundary for IAM resources.  This will be used for tagging and to build out the ARN."
+  type        = string
+  default     = null
+}
+
+variable "use_permissions_boundary" {
+  description = "Whether to use IAM permissions boundary for resources."
+  type        = bool
+  default     = true
+}
+
+variable "key_owner_arns" {
+  description = "ARNS of KMS key owners, needed for use of key"
+  type        = list(string)
+  default     = []
+}
+
+# taken from zarf bb repo
+variable "kms_key_deletion_window" {
+  description = "Waiting period for scheduled KMS Key deletion. Can be 7-30 days."
+  type        = number
+  default     = 7
+}
+
+variable "bucket_configurations" {
+  type = map(object({
+    name           = string
+    service_account = string
+    namespace = string
+  }))
+  default = {
+    loki = {
+      name           = "loki"
+      service_account = "logging-loki"
+      namespace = "logging"
+    }
+    velero = {
+      name           = "velero"
+      service_account = "velero-server"
+      namespace = "velero"
+    }
+  }
+}
+

.github/test-infra/buckets-iac/velero.tf

@@ -0,0 +1,59 @@
+resource "aws_iam_policy" "velero_policy" {
+  name        = "${local.bucket_configurations.velero.name}-irsa-${random_id.unique_id.hex}"
+  path        = "/"
+  description = "Policy to give Velero necessary permissions for cluster backups."
+
+  # Terraform expression result to valid JSON syntax.
+  policy = jsonencode(
+    {
+      Version = "2012-10-17",
+      Statement = [
+        {
+          Effect = "Allow",
+          Action = [
+            "ec2:DescribeVolumes",
+            "ec2:DescribeSnapshots",
+            "ec2:CreateTags",
+            "ec2:CreateVolume",
+            "ec2:CreateSnapshot",
+            "ec2:DeleteSnapshot"
+          ]
+          Resource = [
+            "*"
+          ]
+        },
+        {
+          Effect = "Allow"
+          Action = [
+            "s3:GetObject",
+            "s3:DeleteObject",
+            "s3:PutObject",
+            "s3:AbortMultipartUpload",
+            "s3:ListMultipartUploadParts"
+          ]
+          Resource = [
+            "arn:${data.aws_partition.current.partition}:s3:::${module.S3["velero"].bucket_name}/*"
+          ]
+        },
+        {
+          Effect = "Allow",
+          Action = [
+            "s3:ListBucket"
+          ],
+          Resource = [
+            "arn:${data.aws_partition.current.partition}:s3:::${module.S3["velero"].bucket_name}/*"
+          ]
+        },
+        {
+          Effect = "Allow"
+          Action = [
+            "kms:GenerateDataKey",
+            "kms:Decrypt"
+          ]
+          Resource = [local.kms_key_arns["velero"].kms_key_arn]
+        }
+
+      ]
+  })
+}
+

.github/workflows/nightly-testing.yaml

@@ -0,0 +1,11 @@
+name: Nightly Testing
+
+on:
+  schedule:
+    - cron: '0 6 * * *' # Runs at midnight Mountain every day
+
+jobs:
+  nightly-testing:
+    name: Test Core on EKS
+    uses: ./.github/workflows/test-eks.yaml
+    secrets: inherit