From 4856ec8f708262a687f6a376b467fec4bfb1e232 Mon Sep 17 00:00:00 2001 From: Vijay Chintalapati Date: Wed, 25 Oct 2023 05:02:34 -0700 Subject: [PATCH] feat: Eliminated the use of temporary keys for tests with kubectl in privatelink-access pattern (#1806) Co-authored-by: Bryant Biggs --- .pre-commit-config.yaml | 4 +- .../charts/demo-application/Chart.yaml | 2 +- .../templates/backend-deploy.yaml | 8 +- .../templates/backend-svc.yaml | 6 +- .../templates/client-deploy.yaml | 8 +- .../demo-application/templates/client-ns.yaml | 2 +- .../templates/client-svc.yaml | 4 +- .../templates/frontend-deploy.yaml | 8 +- .../templates/frontend-svc.yaml | 8 +- .../templates/management-ui-deploy.yaml | 10 +- .../templates/management-ui-ns.yaml | 4 +- .../templates/management-ui-svc.yaml | 8 +- .../demo-application/templates/stars-ns.yaml | 2 +- patterns/privatelink-access/README.md | 136 +++++++++--------- patterns/privatelink-access/client.tf | 33 ++++- patterns/privatelink-access/eks.tf | 21 +++ patterns/privatelink-access/outputs.tf | 23 +++ patterns/privatelink-access/versions.tf | 4 + 18 files changed, 185 insertions(+), 106 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69734a100a..2235d24cc7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace args: ['--markdown-linebreak-ext=md'] @@ -10,7 +10,7 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.4 + rev: v1.83.5 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/Chart.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/Chart.yaml index 7a8d00dd4e..7e6d29f9b3 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/Chart.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v2 name: demo-application description: A Helm chart to deploy the demo-application type: application -version: 1.0.0 \ No newline at end of file +version: 1.0.0 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-deploy.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-deploy.yaml index e3c3c1d465..766e0b573a 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-deploy.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-deploy.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: backend + name: backend namespace: stars spec: replicas: 1 @@ -11,10 +11,10 @@ spec: template: metadata: labels: - role: backend + role: backend spec: containers: - - name: backend + - name: backend image: calico/star-probe:v0.1.0 imagePullPolicy: Always command: @@ -22,4 +22,4 @@ spec: - --http-port=6379 - --urls=http://frontend.stars:80/status,http://backend.stars:6379/status,http://client.client:9000/status ports: - - containerPort: 6379 \ No newline at end of file + - containerPort: 6379 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-svc.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-svc.yaml index 5a579569ad..1298c3b74e 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-svc.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-svc.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: Service metadata: - name: backend + name: backend namespace: stars spec: ports: - port: 6379 - targetPort: 6379 + targetPort: 6379 selector: - role: backend \ No newline at end of file + role: backend diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-deploy.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-deploy.yaml index 8d763dcede..35c2d7caab 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-deploy.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-deploy.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: client + name: client namespace: client spec: replicas: 1 @@ -11,14 +11,14 @@ spec: template: metadata: labels: - role: client + role: client spec: containers: - - name: client + - name: client image: calico/star-probe:v0.1.0 imagePullPolicy: Always command: - probe - --urls=http://frontend.stars:80/status,http://backend.stars:6379/status ports: - - containerPort: 9000 \ No newline at end of file + - containerPort: 9000 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-ns.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-ns.yaml index 91f714e9cb..d8e2c02b5a 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-ns.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-ns.yaml @@ -3,4 +3,4 @@ kind: Namespace metadata: name: client labels: - role: client \ No newline at end of file + role: client diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-svc.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-svc.yaml index cfd20be303..a91ded3d1c 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-svc.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-svc.yaml @@ -5,7 +5,7 @@ metadata: namespace: client spec: ports: - - port: 9000 + - port: 9000 targetPort: 9000 selector: - role: client \ No newline at end of file + role: client diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-deploy.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-deploy.yaml index c1b0762c32..5c0352c0d4 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-deploy.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-deploy.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: frontend + name: frontend namespace: stars spec: replicas: 1 @@ -11,10 +11,10 @@ spec: template: metadata: labels: - role: frontend + role: frontend spec: containers: - - name: frontend + - name: frontend image: calico/star-probe:v0.1.0 imagePullPolicy: Always command: @@ -22,4 +22,4 @@ spec: - --http-port=80 - --urls=http://frontend.stars:80/status,http://backend.stars:6379/status,http://client.client:9000/status ports: - - containerPort: 80 \ No newline at end of file + - containerPort: 80 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-svc.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-svc.yaml index 08fafd487b..51426395b7 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-svc.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-svc.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: Service metadata: - name: frontend + name: frontend namespace: stars spec: ports: - - port: 80 - targetPort: 80 + - port: 80 + targetPort: 80 selector: - role: frontend \ No newline at end of file + role: frontend diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-deploy.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-deploy.yaml index 555f338290..391119d3e6 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-deploy.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-deploy.yaml @@ -1,8 +1,8 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: management-ui - namespace: management-ui + name: management-ui + namespace: management-ui spec: replicas: 1 selector: @@ -11,11 +11,11 @@ spec: template: metadata: labels: - role: management-ui + role: management-ui spec: containers: - - name: management-ui + - name: management-ui image: calico/star-collect:v0.1.0 imagePullPolicy: Always ports: - - containerPort: 9001 \ No newline at end of file + - containerPort: 9001 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-ns.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-ns.yaml index ef0a8ec158..afc419b80d 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-ns.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-ns.yaml @@ -1,6 +1,6 @@ apiVersion: v1 kind: Namespace metadata: - name: management-ui + name: management-ui labels: - role: management-ui \ No newline at end of file + role: management-ui diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-svc.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-svc.yaml index 09b850d8d2..9c0e64337d 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-svc.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-svc.yaml @@ -1,12 +1,12 @@ apiVersion: v1 kind: Service metadata: - name: management-ui - namespace: management-ui + name: management-ui + namespace: management-ui spec: type: LoadBalancer ports: - - port: 80 + - port: 80 targetPort: 9001 selector: - role: management-ui \ No newline at end of file + role: management-ui diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/stars-ns.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/stars-ns.yaml index de71efa857..67c4219fe9 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/stars-ns.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/stars-ns.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: stars \ No newline at end of file + name: stars diff --git a/patterns/privatelink-access/README.md b/patterns/privatelink-access/README.md index f7c6e514d8..1124f2012d 100644 --- a/patterns/privatelink-access/README.md +++ b/patterns/privatelink-access/README.md @@ -7,17 +7,37 @@ for further details on `AWS PrivateLink`. ## Deploy -See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and steps to deploy this pattern. +See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and follow the steps below to deploy this pattern. -## Validate +```sh +terraform init +terraform apply -target=module.eventbridge -target=module.nlb --auto-approve +terraform apply --auto-approve +``` -### Network Connectivity +Once the pattern has successfully deployed, you will be provided with multiple +output values. -An output `ssm_test` has been provided to aid in quickly testing the -connectivity from the client EC2 instance to the private EKS cluster via AWS -PrivateLink. Copy the output value and paste it into your terminal to execute -and check the connectivity. If configured correctly, the value returned should -be `ok`. +Review the output value for `cluster_endpoint_private`, it should look similar +to snippet below: + +```sh +aws eks update-cluster-config \ +--region us-west-2 \ +--name privatelink-access \ +--resources-vpc-config endpointPublicAccess=false,endpointPrivateAccess=true +``` + +Copy the command and run it in a terminal session to take cluster API +endpoint private. + +## Test access to EKS Kubernetes API server endpoint + +Of the other output values, the value `ssm_test` is provided to aid in quickly +testing the connectivity from the client EC2 instance to the private EKS cluster +via AWS PrivateLink. Copy the output value, which looks like the snippet shown +below (as an example) and paste it into your terminal to execute and check the +connectivity. If configured correctly, the value returned should be `ok`. ```sh COMMAND="curl -ks https://9A85B21811733524E3ABCDFEA8714642.gr7.us-west-2.eks.amazonaws.com/readyz" @@ -36,78 +56,62 @@ aws ssm get-command-invocation --region us-west-2 \ --output text ``` -### Cluster Access - -To test access to the cluster, you will need to execute Kubernetes API calls -from within the private network to access the cluster. An EC2 instance has been -deployed into a "client" VPC to simulate this scenario. However, since the EKS -cluster was created with your local IAM identity, the `aws-auth` ConfigMap will -only have your local identity that is permitted to access the cluster. Since -cluster's API endpoint is private, we cannot use Terraform to reach it to -add additional entries to the ConfigMap; we can only access the cluster from -within the private network of the cluster's VPC or from the client VPC using AWS -PrivateLink access. - -!!! info - The "client" EC2 instance provided and copying of AWS credentials to - that instance are merely for demonstration purposes only. Please consider - alternate methods of network access such as AWS Client VPN to provide more - secure access. +## Test access to EKS Kubernetes API with `kubectl` Perform the following steps to access the cluster with `kubectl` from the -provided "client" EC2 instance. - -1. Execute the command below on your local machine to get temporary credentials -that will be used on the "client" EC2 instance: +provided Client EC2 instance. - ```sh - aws sts get-session-token --duration-seconds 3600 --output yaml - ``` +### Log into the Client EC2 instance +Start a new SSM session on the Client EC2 instance using the provided +`ssm_start_session` output value. It should look similar to the snippet +shown below. Copy the output value and paste it into your terminal to execute. +Your terminal will now be connected to the Client EC2 instance. -2. Start a new SSM session on the "client" EC2 instance using the provided -`ssm_start_session` output value. Copy the output value and paste it into your -terminal to execute. Your terminal will now be connected to the "client" EC2 -instance. +```sh +aws ssm start-session --region us-west-2 --target i-0280cf604085f4a44 +``` - ```sh - aws ssm start-session --region us-west-2 --target i-0280cf604085f4a44 - ``` +### Update Kubeconfig +On the Client EC2 machine, run the following command to update the local +`~/.kube/config` file to enable access to the cluster: -3. Once logged in, export the following environment variables from the output -of step #1: +```sh +aws eks update-kubeconfig --region us-west-2 --name privatelink-access +``` - !!! warning - The session credentials are only valid for 1 hour; you can - adjust the session duration in the command provided in step #1 +### Test complete access with `kubectl` +Test access by listing the pods running on the cluster: - ```sh - export AWS_ACCESS_KEY_ID=XXXX - export AWS_SECRET_ACCESS_KEY=YYYY - export AWS_SESSION_TOKEN=ZZZZ - ``` +```sh +kubectl get pods -A +``` -4. Run the following command to update the local `~/.kube/config` file to enable -access to the cluster: +```text +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system aws-node-4f8g8 1/1 Running 0 1m +kube-system coredns-6ff9c46cd8-59sqp 1/1 Running 0 1m +kube-system coredns-6ff9c46cd8-svnpb 1/1 Running 0 2m +kube-system kube-proxy-mm2zc 1/1 Running 0 1m +``` - ```sh - aws eks update-kubeconfig --region us-west-2 --name privatelink-access - ``` +## Destroy -5. Test access by listing the pods running on the cluster: +Before we could destroy/teardown all the resources created, we need to ensure +that the cluster state is restored for the Terraform to do a complete cleanup. +This would mean that we make cluster API endpoint public again. - ```sh - kubectl get pods -A - ``` +Review the output value for `cluster_endpoint_public`, it should look similar +to snippet below: - ```text - NAMESPACE NAME READY STATUS RESTARTS AGE - kube-system aws-node-4f8g8 1/1 Running 0 1m - kube-system coredns-6ff9c46cd8-59sqp 1/1 Running 0 1m - kube-system coredns-6ff9c46cd8-svnpb 1/1 Running 0 2m - kube-system kube-proxy-mm2zc 1/1 Running 0 1m - ``` +```sh +aws eks update-cluster-config \ +--region us-west-2 \ +--name privatelink-access \ +--resources-vpc-config endpointPublicAccess=true,endpointPrivateAccess=true +``` -## Destroy +Copy the command and run it in a terminal session to take cluster API +endpoint public. {% include-markdown "../../docs/_partials/destroy.md" diff --git a/patterns/privatelink-access/client.tf b/patterns/privatelink-access/client.tf index 0793c4804f..2852f18f87 100644 --- a/patterns/privatelink-access/client.tf +++ b/patterns/privatelink-access/client.tf @@ -30,7 +30,9 @@ module "client_vpc" { manage_default_security_group = true default_security_group_tags = { Name = "${local.client_name}-default" } - tags = local.tags + tags = merge(local.tags, { + Name = local.client_name + }) } ################################################################################ @@ -44,6 +46,7 @@ module "client_ec2_instance" { create_iam_instance_profile = true iam_role_policies = { + EKSFullAccess = aws_iam_policy.eks_full_access_policy.arn AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } @@ -64,7 +67,29 @@ module "client_ec2_instance" { ./aws/install EOT - tags = local.tags + tags = merge(local.tags, { + Name = local.client_name + }) +} + +resource "aws_iam_policy" "eks_full_access_policy" { + name = "EKSFullAccess" + path = "/" + description = "EKS full-access policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "eks:DescribeCluster", + "eks:ListClusters" + ] + Effect = "Allow" + Resource = module.eks.cluster_arn + }, + ] + }) } module "client_security_group" { @@ -84,5 +109,7 @@ module "client_security_group" { }, ] - tags = local.tags + tags = merge(local.tags, { + Name = local.client_name + }) } diff --git a/patterns/privatelink-access/eks.tf b/patterns/privatelink-access/eks.tf index e698ed35a0..0ac647e023 100644 --- a/patterns/privatelink-access/eks.tf +++ b/patterns/privatelink-access/eks.tf @@ -2,6 +2,18 @@ # EKS Cluster ################################################################################ +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} + module "eks" { source = "terraform-aws-modules/eks/aws" version = "~> 19.16" @@ -9,6 +21,15 @@ module "eks" { cluster_name = local.name cluster_version = "1.27" + cluster_endpoint_public_access = true + manage_aws_auth_configmap = true + + aws_auth_roles = [{ + rolearn = module.client_ec2_instance.iam_role_arn + username = "ec2-client" + groups = ["system:masters"] + }] + cluster_addons = { coredns = {} kube-proxy = {} diff --git a/patterns/privatelink-access/outputs.tf b/patterns/privatelink-access/outputs.tf index 3130bf693f..50f5ac870d 100644 --- a/patterns/privatelink-access/outputs.tf +++ b/patterns/privatelink-access/outputs.tf @@ -6,6 +6,7 @@ output "ssm_start_session" { output "ssm_test" { description = "SSM commands to test connectivity from client EC2 instance to the private EKS cluster" value = <<-EOT + COMMAND="curl -ks ${module.eks.cluster_endpoint}/readyz" COMMAND_ID=$(aws ssm send-command --region ${local.region} \ @@ -22,3 +23,25 @@ output "ssm_test" { --output text EOT } + +output "cluster_endpoint_private" { + description = "Command to set the EKS API server endpoint access private" + value = <<-EOT + + aws eks update-cluster-config \ + --region ${local.region} \ + --name ${module.eks.cluster_name} \ + --resources-vpc-config endpointPublicAccess=false,endpointPrivateAccess=true + EOT +} + +output "cluster_endpoint_public" { + description = "Command to set the EKS API server endpoint access private" + value = <<-EOT + + aws eks update-cluster-config \ + --region ${local.region} \ + --name ${module.eks.cluster_name} \ + --resources-vpc-config endpointPublicAccess=true,endpointPrivateAccess=true + EOT +} diff --git a/patterns/privatelink-access/versions.tf b/patterns/privatelink-access/versions.tf index 4cbe90687e..fb2fa577b1 100644 --- a/patterns/privatelink-access/versions.tf +++ b/patterns/privatelink-access/versions.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/dns" version = ">= 3.0" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20" + } } # ## Used for end-to-end testing on project; update to suit your needs