From ec517ec6c98988876b0e495130c71f337272d66d Mon Sep 17 00:00:00 2001
From: eyalkraft <63912106+eyalkraft@users.noreply.github.com>
Date: Thu, 15 Dec 2022 13:15:24 +0200
Subject: [PATCH] Add eks deployment to terraform (#577)

* add eks

* rename

* fix comments

* add -target doc

* Apply suggestions from code review

Co-authored-by: Oren Zohar <85433724+oren-zohar@users.noreply.github.com>

Co-authored-by: Oren Zohar <85433724+oren-zohar@users.noreply.github.com>
---
 bin/.aws-iam-authenticator-0.5.12.pkg         |  1 +
 bin/aws-iam-authenticator                     |  1 +
 deploy/cloud/.gitignore                       | 27 +++++++++
 deploy/cloud/README.md                        | 58 +++++++++++++++----
 deploy/cloud/main.tf                          | 36 ++++++------
 .../modules/provision-eks-cluster/README.md   |  1 +
 .../provision-eks-cluster/eks-cluster.tf      | 49 ++++++++++++++++
 .../modules/provision-eks-cluster/main.tf     | 19 ++++++
 .../modules/provision-eks-cluster/outputs.tf  | 24 ++++++++
 .../provision-eks-cluster/security-groups.tf  | 29 ++++++++++
 .../provision-eks-cluster/variables.tf        | 11 ++++
 .../modules/provision-eks-cluster/vpc.tf      | 26 +++++++++
 deploy/cloud/outputs.tf                       | 25 ++++++++
 deploy/cloud/terraform.tf                     | 35 +++++++++++
 deploy/cloud/variables.tf                     | 10 +++-
 15 files changed, 321 insertions(+), 31 deletions(-)
 create mode 120000 bin/.aws-iam-authenticator-0.5.12.pkg
 create mode 120000 bin/aws-iam-authenticator
 create mode 100644 deploy/cloud/.gitignore
 create mode 100644 deploy/cloud/modules/provision-eks-cluster/README.md
 create mode 100644 deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf
 create mode 100644 deploy/cloud/modules/provision-eks-cluster/main.tf
 create mode 100644 deploy/cloud/modules/provision-eks-cluster/outputs.tf
 create mode 100644 deploy/cloud/modules/provision-eks-cluster/security-groups.tf
 create mode 100644 deploy/cloud/modules/provision-eks-cluster/variables.tf
 create mode 100644 deploy/cloud/modules/provision-eks-cluster/vpc.tf
 create mode 100644 deploy/cloud/terraform.tf

diff --git a/bin/.aws-iam-authenticator-0.5.12.pkg b/bin/.aws-iam-authenticator-0.5.12.pkg
new file mode 120000
index 0000000000..383f4511d4
--- /dev/null
+++ b/bin/.aws-iam-authenticator-0.5.12.pkg
@@ -0,0 +1 @@
+hermit
\ No newline at end of file
diff --git a/bin/aws-iam-authenticator b/bin/aws-iam-authenticator
new file mode 120000
index 0000000000..02cf6d880a
--- /dev/null
+++ b/bin/aws-iam-authenticator
@@ -0,0 +1 @@
+.aws-iam-authenticator-0.5.12.pkg
\ No newline at end of file
diff --git a/deploy/cloud/.gitignore b/deploy/cloud/.gitignore
new file mode 100644
index 0000000000..6665869f80
--- /dev/null
+++ b/deploy/cloud/.gitignore
@@ -0,0 +1,27 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+*.tfplan
+
+# Crash log files
+crash.log
+
+# Exclude all .tfvars files, which are likely to contain sentitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
diff --git a/deploy/cloud/README.md b/deploy/cloud/README.md
index 7b265504c9..99010e3384 100644
--- a/deploy/cloud/README.md
+++ b/deploy/cloud/README.md
@@ -1,40 +1,74 @@
 # Cloud Deployment
 
-**Motivation**
-Provide an easy and deterministic way to setup latest cloud environment so it can be monitored and used properly.
+**Motivation**  
+Provide an easy and deterministic way to set up latest cloud environment, so it can be monitored and used properly.
+
+This guide deploys both an Elastic cloud environment, and an AWS EKS cluster. To only deploy specific resources, check out the examples section.
 
 **Prerequisite**
-* [terraform](https://www.terraform.io/)
+* [Terraform](https://developer.hashicorp.com/terraform/downloads)
+* the AWS CLI, [installed](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) and [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
+* [AWS IAM Authenticator](https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html)
+* the [Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl/), also known as `kubectl`
+
 
 **How To**
 Create environment
 1. Create an [API token](https://cloud.elastic.co/deployment-features/keys) from your cloud console account.
+
     1.1 use the token `export EC_API_KEY={TOKEN}`
+
 2. run `cd deploy/cloud`
 3. run `terraform init`
-4. run `terraform apply --auto-approve` to create the environment from the latest version (the latest version is vary in cloud/regions combinations).
-To connect to the environment use the console ui or see the details how to connect to the environment, use `terraform output -json`
+2. to create the environment from the latest version (the latest version is varying in cloud/regions combinations).
+   ```bash
+   cd deploy/cloud
+   terraform init
+   terraform apply --auto-approve
+3. Run the following command to retrieve the access credentials for your EKS cluster and configure kubectl.
+   ```bash
+   aws eks --region $(terraform output -raw eks_region) update-kubeconfig \
+       --name $(terraform output -raw eks_cluster_name)
 
-Delete environment
-1. `terraform destroy --auto-approve`
+To connect to the environment use the console UI or see the details how to connect to the environment, using:
+```bash
+terraform output -json
+**Delete environment:**
+```bash
+terraform destroy --auto-approve
 
 **Next Steps**
 * [Setup](https://github.com/elastic/security-team/blob/main/docs/cloud-security-posture-team/onboarding/deploy-agent-cloudbeat-on-eks.mdx) EKS cluster
-* Setup Vanila cluster
-* Enable rules add slack webhook to connctor
+* Setup Self-Managed cluster
+* Enable rules add slack webhook to connector
 
 # Examples
 
 ## Specific version
-To create an environment with specific version use 
+To create an environment with specific version use
 
 `terraform apply --auto-approve -var="stack_version=8.5.1"`
 When working with non production versions it is required to also update the deployment regions.
-For example, to deploy `8.6.0-SNAPSHOT` use 
+For example, to deploy `8.6.0-SNAPSHOT` use
 
 `terraform apply --auto-approve -var="stack_version=8.6.0-SNAPSHOT" -var="ess_region=gcp-us-west2"`
 
 ## Named environment
-To give your environment a different prefix in the name use 
+To give your environment a different prefix in the name use
 
 `terraform apply --auto-approve -var="deployment_name_prefix=elastic-deployment"`
+
+## Deploy specific resources
+To deploy specific resources use the `-target` flag.
+
+### Deploy only Elastic Cloud with no EKS cluster or Dashboard
+
+`terraform apply --auto-approve -target "module.ec_deployment"`
+
+### Deploy only Dashboard on an existing Elastic Cloud deployment
+
+`terraform apply --auto-approve -target "null_resource.rules" -target "null_resource.store_local_dashboard"`
+
+### Deploy only EKS cluster
+
+`terraform apply --auto-approve -target "module.eks"`
diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf
index 318c087bd3..bb987117a8 100644
--- a/deploy/cloud/main.tf
+++ b/deploy/cloud/main.tf
@@ -1,24 +1,14 @@
-terraform {
-  required_version = ">= 1.1.8, < 2.0.0"
-  required_providers {
-    ec = {
-      source  = "elastic/ec"
-      version = ">=0.5.0"
-    }
-  }
-}
-
 provider "ec" {}
 
 
-module "ec_deployment"  {
+module "ec_deployment" {
   source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment"
 
   region        = var.ess_region
   stack_version = var.stack_version
 
   deployment_template    = var.deployment_template
-  deployment_name_prefix = var.deployment_name_prefix
+  deployment_name_prefix = "${var.deployment_name_prefix}-${random_string.suffix.result}"
 
   integrations_server = true
 
@@ -33,8 +23,15 @@ module "ec_deployment"  {
   }
 }
 
+module "eks" {
+  source = "./modules/provision-eks-cluster"
+
+  region              = var.eks_region
+  cluster_name_prefix = "${var.deployment_name_prefix}-${random_string.suffix.result}"
+}
+
 data "local_file" "dashboard" {
-    filename = "data/dashboard.ndjson"
+  filename = "data/dashboard.ndjson"
 }
 
 resource "null_resource" "store_local_dashboard" {
@@ -42,14 +39,14 @@ resource "null_resource" "store_local_dashboard" {
     command = "curl -X POST -u ${module.ec_deployment.elasticsearch_username}:${module.ec_deployment.elasticsearch_password} ${module.ec_deployment.kibana_url}/api/saved_objects/_import?overwrite=true -H \"kbn-xsrf: true\" --form file=@data/dashboard.ndjson"
   }
   depends_on = [module.ec_deployment]
-  triggers = {
+  triggers   = {
     dashboard_sha1 = sha1(file("data/dashboard.ndjson"))
   }
 }
 
 
 data "local_file" "rules" {
-    filename = "data/rules.ndjson"
+  filename = "data/rules.ndjson"
 }
 
 resource "null_resource" "rules" {
@@ -57,7 +54,12 @@ resource "null_resource" "rules" {
     command = "curl -X POST -u ${module.ec_deployment.elasticsearch_username}:${module.ec_deployment.elasticsearch_password} ${module.ec_deployment.kibana_url}/api/saved_objects/_import?overwrite=true -H \"kbn-xsrf: true\" --form file=@data/rules.ndjson"
   }
   depends_on = [module.ec_deployment]
-  triggers = {
-    dashboard_sha1 = "${sha1(file("data/rules.ndjson"))}"
+  triggers   = {
+    dashboard_sha1 = sha1(file("data/rules.ndjson"))
   }
 }
+
+resource "random_string" "suffix" {
+  length  = 3
+  special = false
+}
diff --git a/deploy/cloud/modules/provision-eks-cluster/README.md b/deploy/cloud/modules/provision-eks-cluster/README.md
new file mode 100644
index 0000000000..81229783e9
--- /dev/null
+++ b/deploy/cloud/modules/provision-eks-cluster/README.md
@@ -0,0 +1 @@
+Based on the [Provision an EKS Cluster tutorial](https://developer.hashicorp.com/terraform/tutorials/kubernetes/eks), and [repo](https://github.com/hashicorp/learn-terraform-provision-eks-cluster).
diff --git a/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf b/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf
new file mode 100644
index 0000000000..7b8b81304b
--- /dev/null
+++ b/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf
@@ -0,0 +1,49 @@
+module "eks" {
+  source  = "terraform-aws-modules/eks/aws"
+  version = "18.26.6"
+
+  cluster_name    = local.cluster_name
+  cluster_version = "1.24"
+
+  vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
+
+  eks_managed_node_group_defaults = {
+    ami_type = "AL2_x86_64"
+
+    attach_cluster_primary_security_group = true
+
+    # Disabling and using externally provided security groups
+    create_security_group = false
+  }
+
+  eks_managed_node_groups = {
+    one = {
+      name = "${var.cluster_name_prefix}-1"
+
+      instance_types = ["t3.small"]
+
+      min_size     = 1
+      max_size     = 3
+      desired_size = 2
+
+      vpc_security_group_ids = [
+        aws_security_group.node_group_one.id
+      ]
+    }
+
+    two = {
+      name = "${var.cluster_name_prefix}-2"
+
+      instance_types = ["t3.medium"]
+
+      min_size     = 1
+      max_size     = 2
+      desired_size = 1
+
+      vpc_security_group_ids = [
+        aws_security_group.node_group_two.id
+      ]
+    }
+  }
+}
diff --git a/deploy/cloud/modules/provision-eks-cluster/main.tf b/deploy/cloud/modules/provision-eks-cluster/main.tf
new file mode 100644
index 0000000000..5a73c27dc1
--- /dev/null
+++ b/deploy/cloud/modules/provision-eks-cluster/main.tf
@@ -0,0 +1,19 @@
+# Kubernetes provider
+# https://learn.hashicorp.com/terraform/kubernetes/provision-eks-cluster#optional-configure-terraform-kubernetes-provider
+# To learn how to schedule deployments and services using the provider, go here: https://learn.hashicorp.com/terraform/kubernetes/deploy-nginx-kubernetes
+# The Kubernetes provider is included in this file so the EKS module can complete successfully. Otherwise, it throws an error when creating `kubernetes_config_map.aws_auth`.
+# You should **not** schedule deployments and services in this workspace. This keeps workspaces modular (one for provision EKS, another for scheduling Kubernetes resources) as per best practices.
+provider "kubernetes" {
+  host                   = module.eks.cluster_endpoint
+  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
+}
+
+provider "aws" {
+  region = var.region
+}
+
+data "aws_availability_zones" "available" {}
+
+locals {
+  cluster_name = var.cluster_name_prefix
+}
diff --git a/deploy/cloud/modules/provision-eks-cluster/outputs.tf b/deploy/cloud/modules/provision-eks-cluster/outputs.tf
new file mode 100644
index 0000000000..8f5004a159
--- /dev/null
+++ b/deploy/cloud/modules/provision-eks-cluster/outputs.tf
@@ -0,0 +1,24 @@
+output "cluster_id" {
+  description = "EKS cluster ID"
+  value       = module.eks.cluster_id
+}
+
+output "cluster_endpoint" {
+  description = "Endpoint for EKS control plane"
+  value       = module.eks.cluster_endpoint
+}
+
+output "cluster_security_group_id" {
+  description = "Security group ids attached to the cluster control plane"
+  value       = module.eks.cluster_security_group_id
+}
+
+output "region" {
+  description = "AWS region"
+  value       = var.region
+}
+
+output "cluster_name" {
+  description = "Kubernetes Cluster Name"
+  value       = local.cluster_name
+}
diff --git a/deploy/cloud/modules/provision-eks-cluster/security-groups.tf b/deploy/cloud/modules/provision-eks-cluster/security-groups.tf
new file mode 100644
index 0000000000..6c2e4014af
--- /dev/null
+++ b/deploy/cloud/modules/provision-eks-cluster/security-groups.tf
@@ -0,0 +1,29 @@
+resource "aws_security_group" "node_group_one" {
+  name_prefix = "${var.cluster_name_prefix}_1"
+  vpc_id      = module.vpc.vpc_id
+
+  ingress {
+    from_port = 22
+    to_port   = 22
+    protocol  = "tcp"
+
+    cidr_blocks = [
+      "10.0.0.0/8",
+    ]
+  }
+}
+
+resource "aws_security_group" "node_group_two" {
+  name_prefix = "${var.cluster_name_prefix}_2"
+  vpc_id      = module.vpc.vpc_id
+
+  ingress {
+    from_port = 22
+    to_port   = 22
+    protocol  = "tcp"
+
+    cidr_blocks = [
+      "192.168.0.0/16",
+    ]
+  }
+}
diff --git a/deploy/cloud/modules/provision-eks-cluster/variables.tf b/deploy/cloud/modules/provision-eks-cluster/variables.tf
new file mode 100644
index 0000000000..4d89e963e6
--- /dev/null
+++ b/deploy/cloud/modules/provision-eks-cluster/variables.tf
@@ -0,0 +1,11 @@
+variable "region" {
+  description = "AWS region"
+  type        = string
+  default     = "eu-west-1"
+}
+
+variable "cluster_name_prefix" {
+  description = "EKS cluster name prefix"
+  type        = string
+  default     = "cloudbeat-tf"
+}
diff --git a/deploy/cloud/modules/provision-eks-cluster/vpc.tf b/deploy/cloud/modules/provision-eks-cluster/vpc.tf
new file mode 100644
index 0000000000..395b981e9a
--- /dev/null
+++ b/deploy/cloud/modules/provision-eks-cluster/vpc.tf
@@ -0,0 +1,26 @@
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "3.14.2"
+
+  name = "${var.cluster_name_prefix}-vpc"
+
+  cidr = "10.0.0.0/16"
+  azs  = slice(data.aws_availability_zones.available.names, 0, 3)
+
+  private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+  public_subnets  = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
+
+  enable_nat_gateway   = true
+  single_nat_gateway   = true
+  enable_dns_hostnames = true
+
+  public_subnet_tags = {
+    "kubernetes.io/cluster/${local.cluster_name}" = "shared"
+    "kubernetes.io/role/elb"                      = 1
+  }
+
+  private_subnet_tags = {
+    "kubernetes.io/cluster/${local.cluster_name}" = "shared"
+    "kubernetes.io/role/internal-elb"             = 1
+  }
+}
diff --git a/deploy/cloud/outputs.tf b/deploy/cloud/outputs.tf
index 8a35fb7244..4246cf7a58 100644
--- a/deploy/cloud/outputs.tf
+++ b/deploy/cloud/outputs.tf
@@ -24,3 +24,28 @@ output "admin_console_url" {
   value       = module.ec_deployment.admin_console_url
   description = "The admin console URL"
 }
+
+output "eks_cluster_id" {
+  description = "EKS cluster ID"
+  value       = module.eks.cluster_id
+}
+
+output "eks_cluster_endpoint" {
+  description = "Endpoint for EKS control plane"
+  value       = module.eks.cluster_endpoint
+}
+
+output "eks_cluster_security_group_id" {
+  description = "Security group ids attached to the cluster control plane"
+  value       = module.eks.cluster_security_group_id
+}
+
+output "eks_region" {
+  description = "AWS region"
+  value       = module.eks.region
+}
+
+output "eks_cluster_name" {
+  description = "Kubernetes Cluster Name"
+  value       = module.eks.cluster_name
+}
diff --git a/deploy/cloud/terraform.tf b/deploy/cloud/terraform.tf
new file mode 100644
index 0000000000..8a988729f6
--- /dev/null
+++ b/deploy/cloud/terraform.tf
@@ -0,0 +1,35 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.15.0"
+    }
+
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.1.0"
+    }
+
+    tls = {
+      source  = "hashicorp/tls"
+      version = "~> 3.4.0"
+    }
+
+    cloudinit = {
+      source  = "hashicorp/cloudinit"
+      version = "~> 2.2.0"
+    }
+
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.12.1"
+    }
+
+    ec = {
+      source  = "elastic/ec"
+      version = ">=0.5.0"
+    }
+  }
+
+  required_version = ">= 1.3, <2.0.0"
+}
diff --git a/deploy/cloud/variables.tf b/deploy/cloud/variables.tf
index 897c0538f8..0607269290 100644
--- a/deploy/cloud/variables.tf
+++ b/deploy/cloud/variables.tf
@@ -54,6 +54,12 @@ variable "security_team_repository" {
 }
 
 variable "deployment_name_prefix" {
-  default = "cloudbeat"
-  description = "Optional set a prefix of the deployment"
+  default = "cloudbeat-tf"
+  description = "Optional set a prefix of the deployment. Defaults to cloudbeat-tf"
+}
+
+variable "eks_region" {
+  default     = "eu-west-1"
+  description = "Optional AWS region where the EKS cluster will be created. Defaults to eu-west-1"
+  type        = string
 }