Merge branch 'main' into health-status-agent-v2

bin/.aws-iam-authenticator-0.5.12.pkg

@@ -0,0 +1 @@
+hermit

bin/aws-iam-authenticator

@@ -0,0 +1 @@
+.aws-iam-authenticator-0.5.12.pkg

config/benchmark.go

@@ -1,3 +1,23 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Config is put into a different package to prevent cyclic imports in case
+// it is needed in several locations
+
 package config
 
 // https://github.com/elastic/integrations/tree/main/packages/cloud_security_posture/data_stream/findings/agent/stream

config/config.go

@@ -48,15 +48,14 @@ type Fetcher struct {
 }
 
 type Config struct {
-	Type       string                  `config:"type"`
 	AWSConfig  aws.ConfigAWS           `config:",inline"`
 	RuntimeCfg *RuntimeConfig          `config:"runtime_cfg"`
 	Fetchers   []*config.C             `config:"fetchers"`
 	KubeConfig string                  `config:"kube_config"`
 	Period     time.Duration           `config:"period"`
 	Processors processors.PluginConfig `config:"processors"`
 	BundlePath string                  `config:"bundle_path"`
-	Benchmark  *string                 `config:"config.v1.benchmark"`
+	Benchmark  string                  `config:"config.v1.benchmark"`
 }
 
 type RuntimeConfig struct {
@@ -79,23 +78,18 @@ func New(cfg *config.C) (*Config, error) {
 		return nil, err
 	}
 
-	if c.Benchmark != nil {
-		if !isSupportedBenchmark(*c.Benchmark) {
+	if c.Benchmark != "" {
+		if !isSupportedBenchmark(c.Benchmark) {
 			return c, ErrBenchmarkNotSupported
 		}
-		c.Type = buildConfigType(*c.Benchmark)
-	} else {
-		if c.RuntimeCfg != nil && c.RuntimeCfg.ActivatedRules != nil && len(c.RuntimeCfg.ActivatedRules.CisEks) > 0 {
-			c.Type = buildConfigType(CIS_EKS)
-		}
 	}
 	return c, nil
 }
 
 func defaultConfig() (*Config, error) {
 	ret := &Config{
-		Period: 4 * time.Hour,
-		Type:   buildConfigType(CIS_K8S),
+		Period:    4 * time.Hour,
+		Benchmark: CIS_K8S,
 	}
 
 	bundle, err := getBundlePath()

config/config_test.go

@@ -122,7 +122,7 @@ fetchers:
 			c, err := New(cfg)
 			s.NoError(err)
 
-			s.Equal(test.expectedType, c.Type)
+			s.Equal(test.expectedType, c.Benchmark)
 			s.EqualValues(test.expectedActivatedRules, c.RuntimeCfg.ActivatedRules)
 			s.Equal(test.expectedAWSConfig, c.AWSConfig)
 			s.Equal(test.expectedFetchers, len(c.Fetchers))
@@ -207,7 +207,7 @@ config:
 				return
 			}
 			s.NoError(err)
-			s.Equal(test.expected, *c.Benchmark)
+			s.Equal(test.expected, c.Benchmark)
 		})
 	}
 }
@@ -322,7 +322,7 @@ runtime_cfg:
 			c, err := New(cfg)
 			s.NoError(err)
 
-			s.Equal(test.expectedType, c.Type)
+			s.Equal(test.expectedType, c.Benchmark)
 			s.Equal(test.expectedActivatedRules, c.RuntimeCfg.ActivatedRules.CisK8s)
 			s.Equal(test.expectedEksActivatedRules, c.RuntimeCfg.ActivatedRules.CisEks)
 		})

deploy/cloud/.gitignore

@@ -0,0 +1,27 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+*.tfplan
+
+# Crash log files
+crash.log
+
+# Exclude all .tfvars files, which are likely to contain sentitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

deploy/cloud/README.md

@@ -1,40 +1,74 @@
 # Cloud Deployment
 
-**Motivation**
-Provide an easy and deterministic way to setup latest cloud environment so it can be monitored and used properly.
+**Motivation**  
+Provide an easy and deterministic way to set up latest cloud environment, so it can be monitored and used properly.
+
+This guide deploys both an Elastic cloud environment, and an AWS EKS cluster. To only deploy specific resources, check out the examples section.
 
 **Prerequisite**
-* [terraform](https://www.terraform.io/)
+* [Terraform](https://developer.hashicorp.com/terraform/downloads)
+* the AWS CLI, [installed](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) and [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
+* [AWS IAM Authenticator](https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html)
+* the [Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl/), also known as `kubectl`
+
 
 **How To**
 Create environment
 1. Create an [API token](https://cloud.elastic.co/deployment-features/keys) from your cloud console account.
+
     1.1 use the token `export EC_API_KEY={TOKEN}`
+
 2. run `cd deploy/cloud`
 3. run `terraform init`
-4. run `terraform apply --auto-approve` to create the environment from the latest version (the latest version is vary in cloud/regions combinations).
-To connect to the environment use the console ui or see the details how to connect to the environment, use `terraform output -json`
+2. to create the environment from the latest version (the latest version is varying in cloud/regions combinations).
+   ```bash
+   cd deploy/cloud
+   terraform init
+   terraform apply --auto-approve
+3. Run the following command to retrieve the access credentials for your EKS cluster and configure kubectl.
+   ```bash
+   aws eks --region $(terraform output -raw eks_region) update-kubeconfig \
+       --name $(terraform output -raw eks_cluster_name)
 
-Delete environment
-1. `terraform destroy --auto-approve`
+To connect to the environment use the console UI or see the details how to connect to the environment, using:
+```bash
+terraform output -json
+**Delete environment:**
+```bash
+terraform destroy --auto-approve
 
 **Next Steps**
 * [Setup](https://github.com/elastic/security-team/blob/main/docs/cloud-security-posture-team/onboarding/deploy-agent-cloudbeat-on-eks.mdx) EKS cluster
-* Setup Vanila cluster
-* Enable rules add slack webhook to connctor
+* Setup Self-Managed cluster
+* Enable rules add slack webhook to connector
 
 # Examples
 
 ## Specific version
-To create an environment with specific version use 
+To create an environment with specific version use
 
 `terraform apply --auto-approve -var="stack_version=8.5.1"`
 When working with non production versions it is required to also update the deployment regions.
-For example, to deploy `8.6.0-SNAPSHOT` use 
+For example, to deploy `8.6.0-SNAPSHOT` use
 
 `terraform apply --auto-approve -var="stack_version=8.6.0-SNAPSHOT" -var="ess_region=gcp-us-west2"`
 
 ## Named environment
-To give your environment a different prefix in the name use 
+To give your environment a different prefix in the name use
 
 `terraform apply --auto-approve -var="deployment_name_prefix=elastic-deployment"`
+
+## Deploy specific resources
+To deploy specific resources use the `-target` flag.
+
+### Deploy only Elastic Cloud with no EKS cluster or Dashboard
+
+`terraform apply --auto-approve -target "module.ec_deployment"`
+
+### Deploy only Dashboard on an existing Elastic Cloud deployment
+
+`terraform apply --auto-approve -target "null_resource.rules" -target "null_resource.store_local_dashboard"`
+
+### Deploy only EKS cluster
+
+`terraform apply --auto-approve -target "module.eks"`

deploy/cloud/main.tf

@@ -1,24 +1,14 @@
-terraform {
-  required_version = ">= 1.1.8, < 2.0.0"
-  required_providers {
-    ec = {
-      source  = "elastic/ec"
-      version = ">=0.5.0"
-    }
-  }
-}
-
 provider "ec" {}
 
 
-module "ec_deployment"  {
+module "ec_deployment" {
   source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment"
 
   region        = var.ess_region
   stack_version = var.stack_version
 
   deployment_template    = var.deployment_template
-  deployment_name_prefix = var.deployment_name_prefix
+  deployment_name_prefix = "${var.deployment_name_prefix}-${random_string.suffix.result}"
 
   integrations_server = true
 
@@ -33,31 +23,43 @@ module "ec_deployment"  {
   }
 }
 
+module "eks" {
+  source = "./modules/provision-eks-cluster"
+
+  region              = var.eks_region
+  cluster_name_prefix = "${var.deployment_name_prefix}-${random_string.suffix.result}"
+}
+
 data "local_file" "dashboard" {
-    filename = "data/dashboard.ndjson"
+  filename = "data/dashboard.ndjson"
 }
 
 resource "null_resource" "store_local_dashboard" {
   provisioner "local-exec" {
     command = "curl -X POST -u ${module.ec_deployment.elasticsearch_username}:${module.ec_deployment.elasticsearch_password} ${module.ec_deployment.kibana_url}/api/saved_objects/_import?overwrite=true -H \"kbn-xsrf: true\" --form file=@data/dashboard.ndjson"
   }
   depends_on = [module.ec_deployment]
-  triggers = {
+  triggers   = {
     dashboard_sha1 = sha1(file("data/dashboard.ndjson"))
   }
 }
 
 
 data "local_file" "rules" {
-    filename = "data/rules.ndjson"
+  filename = "data/rules.ndjson"
 }
 
 resource "null_resource" "rules" {
   provisioner "local-exec" {
     command = "curl -X POST -u ${module.ec_deployment.elasticsearch_username}:${module.ec_deployment.elasticsearch_password} ${module.ec_deployment.kibana_url}/api/saved_objects/_import?overwrite=true -H \"kbn-xsrf: true\" --form file=@data/rules.ndjson"
   }
   depends_on = [module.ec_deployment]
-  triggers = {
-    dashboard_sha1 = "${sha1(file("data/rules.ndjson"))}"
+  triggers   = {
+    dashboard_sha1 = sha1(file("data/rules.ndjson"))
   }
 }
+
+resource "random_string" "suffix" {
+  length  = 3
+  special = false
+}

deploy/cloud/modules/provision-eks-cluster/README.md

@@ -0,0 +1 @@
+Based on the [Provision an EKS Cluster tutorial](https://developer.hashicorp.com/terraform/tutorials/kubernetes/eks), and [repo](https://github.com/hashicorp/learn-terraform-provision-eks-cluster).

deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf

@@ -0,0 +1,49 @@
+module "eks" {
+  source  = "terraform-aws-modules/eks/aws"
+  version = "18.26.6"
+
+  cluster_name    = local.cluster_name
+  cluster_version = "1.24"
+
+  vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
+
+  eks_managed_node_group_defaults = {
+    ami_type = "AL2_x86_64"
+
+    attach_cluster_primary_security_group = true
+
+    # Disabling and using externally provided security groups
+    create_security_group = false
+  }
+
+  eks_managed_node_groups = {
+    one = {
+      name = "${var.cluster_name_prefix}-1"
+
+      instance_types = ["t3.small"]
+
+      min_size     = 1
+      max_size     = 3
+      desired_size = 2
+
+      vpc_security_group_ids = [
+        aws_security_group.node_group_one.id
+      ]
+    }
+
+    two = {
+      name = "${var.cluster_name_prefix}-2"
+
+      instance_types = ["t3.medium"]
+
+      min_size     = 1
+      max_size     = 2
+      desired_size = 1
+
+      vpc_security_group_ids = [
+        aws_security_group.node_group_two.id
+      ]
+    }
+  }
+}

deploy/cloud/modules/provision-eks-cluster/main.tf

@@ -0,0 +1,19 @@
+# Kubernetes provider
+# https://learn.hashicorp.com/terraform/kubernetes/provision-eks-cluster#optional-configure-terraform-kubernetes-provider
+# To learn how to schedule deployments and services using the provider, go here: https://learn.hashicorp.com/terraform/kubernetes/deploy-nginx-kubernetes
+# The Kubernetes provider is included in this file so the EKS module can complete successfully. Otherwise, it throws an error when creating `kubernetes_config_map.aws_auth`.
+# You should **not** schedule deployments and services in this workspace. This keeps workspaces modular (one for provision EKS, another for scheduling Kubernetes resources) as per best practices.
+provider "kubernetes" {
+  host                   = module.eks.cluster_endpoint
+  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
+}
+
+provider "aws" {
+  region = var.region
+}
+
+data "aws_availability_zones" "available" {}
+
+locals {
+  cluster_name = var.cluster_name_prefix
+}

deploy/cloud/modules/provision-eks-cluster/outputs.tf

@@ -0,0 +1,24 @@
+output "cluster_id" {
+  description = "EKS cluster ID"
+  value       = module.eks.cluster_id
+}
+
+output "cluster_endpoint" {
+  description = "Endpoint for EKS control plane"
+  value       = module.eks.cluster_endpoint
+}
+
+output "cluster_security_group_id" {
+  description = "Security group ids attached to the cluster control plane"
+  value       = module.eks.cluster_security_group_id
+}
+
+output "region" {
+  description = "AWS region"
+  value       = var.region
+}
+
+output "cluster_name" {
+  description = "Kubernetes Cluster Name"
+  value       = local.cluster_name
+}