From edb8492aad00d1525d0691f5bdba7bbd66f18016 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 12 Sep 2023 17:02:13 -0400 Subject: [PATCH 1/2] feat: Update Wireguard example to new layout --- .github/scripts/plan-examples.py | 22 +- patterns/wireguard-with-cilium/README.md | 150 ++++++++++-- patterns/wireguard-with-cilium/eks.tf | 102 +++++++++ patterns/wireguard-with-cilium/main.tf | 238 +++----------------- patterns/wireguard-with-cilium/outputs.tf | 4 - patterns/wireguard-with-cilium/sample.tf | 83 +++++++ patterns/wireguard-with-cilium/variables.tf | 6 - patterns/wireguard-with-cilium/versions.tf | 33 --- 8 files changed, 352 insertions(+), 286 deletions(-) create mode 100644 patterns/wireguard-with-cilium/eks.tf delete mode 100644 patterns/wireguard-with-cilium/outputs.tf create mode 100644 patterns/wireguard-with-cilium/sample.tf delete mode 100644 patterns/wireguard-with-cilium/variables.tf delete mode 100644 patterns/wireguard-with-cilium/versions.tf diff --git a/.github/scripts/plan-examples.py b/.github/scripts/plan-examples.py index 2be720a15b..e05d8b1a9d 100644 --- a/.github/scripts/plan-examples.py +++ b/.github/scripts/plan-examples.py @@ -3,23 +3,23 @@ import re -def get_examples(): +def get_patterns(): """ - Get all Terraform example root directories using their respective `versions.tf`; + Get all pattern root directories using their respective `main.tf`; returning a string formatted json array of the example directories minus those that are excluded """ exclude = { - 'examples/appmesh-mtls', # excluded until Rout53 is setup - 'examples/blue-green-upgrade/environment', - 'examples/blue-green-upgrade/modules/eks_cluster', - 'examples/istio-multi-cluster/1.cluster1', # relies on remote state - 'examples/istio-multi-cluster/2.cluster2', # relies on remote state - 'examples/privatelink-access', + 'patterns/appmesh-mtls', # excluded until Rout53 is setup + 'patterns/blue-green-upgrade/environment', + 'patterns/blue-green-upgrade/modules/eks_cluster', + 'patterns/istio-multi-cluster/1.cluster1', # relies on remote state + 'patterns/istio-multi-cluster/2.cluster2', # relies on remote state + 'patterns/privatelink-access', } projects = { - x.replace('/versions.tf', '') - for x in glob.glob('patterns/**/versions.tf', recursive=True) + x.replace('/main.tf', '') + for x in glob.glob('patterns/**/main.tf', recursive=True) if not re.match(r'^.+/_', x) } @@ -27,4 +27,4 @@ def get_examples(): if __name__ == '__main__': - get_examples() + get_patterns() diff --git a/patterns/wireguard-with-cilium/README.md b/patterns/wireguard-with-cilium/README.md index 877b58ab79..d799203f41 100644 --- a/patterns/wireguard-with-cilium/README.md +++ b/patterns/wireguard-with-cilium/README.md @@ -1,9 +1,16 @@ # Transparent Encryption with Cilium and Wireguard -This pattern demonstrates Cilium configured in CNI chaining mode with VPC CNI and with Wireguard transparent encryption enabled on an Amazon EKS cluster. +This pattern demonstrates Cilium configured in CNI chaining mode with the VPC CNI and with Wireguard transparent encryption enabled on an Amazon EKS cluster. -- [Cilium CNI Chaining Documentation](https://docs.cilium.io/en/v1.12/gettingstarted/cni-chaining-aws-cni/) -- [Cilium Wireguard Encryption Documentation](https://docs.cilium.io/en/v1.12/gettingstarted/encryption-wireguard/) +- [Cilium CNI Chaining Documentation](https://docs.cilium.io/en/stable/installation/cni-chaining-aws-cni/) +- [Cilium Wireguard Encryption Documentation](https://docs.cilium.io/en/stable/security/network/encryption-wireguard/) + +## Areas of Interest + +- `eks.tf` contains the cluster configuration and the deployment of Cilium. + - There are no specific requirements from an EKS perspective, other than the Linux Kernel version used by the OS must be 5.10+. + On Amazon EKS, this is available starting with EKS 1.24, or users can utilize the Bottlerocket OS for EKS < 1.23 +- `sample.tf` provides a sample application used to demonstrate the encrypted connectivity. This is optional and not required for the pattern. ## Deploy @@ -11,17 +18,39 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started ## Validate -1. List the daemonsets +1. Get the Cilium status from one of the Cilium pods. + + Under the `Encryption` field, it should state `Wireguard` with a PubKey. + `NodeEncryption: Disabled` is expected since `NodeEncryption` was not enabled + via the Helm values provided. ```sh - kubectl get ds -n kube-system + kubectl -n kube-system exec -ti ds/cilium -- cilium status ``` ```text - NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE - aws-node 2 2 2 2 2 156m - cilium 2 2 2 2 2 kubernetes.io/os=linux 152m - kube-proxy 2 2 2 2 2 156m + Defaulted container "cilium-agent" out of: cilium-agent, config (init), mount-cgroup (init), apply-sysctl-overwrites (init), mount-bpf-fs (init), clean-cilium-state (init), install-cni-binaries (init) + KVStore: Ok Disabled + Kubernetes: Ok 1.27+ (v1.27.4-eks-2d98532) [linux/amd64] + Kubernetes APIs: ["EndpointSliceOrEndpoint", "cilium/v2::CiliumClusterwideNetworkPolicy", "cilium/v2::CiliumEndpoint", "cilium/v2::CiliumNetworkPolicy", "cilium/v2::CiliumNode", "cilium/v2alpha1::CiliumCIDRGroup", "core/v1::Namespace", "core/v1::Pods", "core/v1::Service", "networking.k8s.io/v1::NetworkPolicy"] + KubeProxyReplacement: False [eth0 10.0.45.128 (Direct Routing), eth1 10.0.40.206] + Host firewall: Disabled + CNI Chaining: aws-cni + Cilium: Ok 1.14.1 (v1.14.1-c191ef6f) + NodeMonitor: Listening for events on 2 CPUs with 64x4096 of shared memory + Cilium health daemon: Ok + IPAM: IPv4: 1/254 allocated from 10.0.1.0/24, + IPv4 BIG TCP: Disabled + IPv6 BIG TCP: Disabled + BandwidthManager: Disabled + Host Routing: Legacy + Masquerading: Disabled + Controller Status: 20/20 healthy + Proxy Status: No managed proxy redirect + Global Identity Range: min 256, max 65535 + Hubble: Ok Current/Max Flows: 4095/4095 (100.00%), Flows/s: 1.58 Metrics: Disabled + Encryption: Wireguard [NodeEncryption: Disabled, cilium_wg0 (Pubkey: Es25c2idJtRzE0/FKAOvKPJ7ybRmZ23KrufK3HOuZTY=, Port: 51871, Peers: 1)] + Cluster health: Probe disabled ``` 2. Open a shell inside the cilium container @@ -30,39 +59,114 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started kubectl -n kube-system exec -ti ds/cilium -- bash ``` -3. Verify Encryption is enabled - - ```sh - cilium status | grep Encryption - ``` - - ```text - Encryption: Wireguard [cilium_wg0 (Pubkey: b2krgbHgaCsVWALMnFLiS/RekhhcE36PXEjQ7T8+mW0=, Port: 51871, Peers: 1)] - ``` - -4. Install [`tcpdump`](https://www.tcpdump.org/) +3. Install [`tcpdump`](https://www.tcpdump.org/) ```sh apt-get update apt-get install -y tcpdump ``` -5. Start a packet capture on `cilium_wg0` and verify you see payload in clear text, it means the traffic is encrypted with wireguard +4. Start a packet capture on `cilium_wg0` and verify you see payload in clear text, it means the traffic is encrypted with wireguard ```sh tcpdump -A -c 40 -i cilium_wg0 | grep "Welcome to nginx!" ``` ```text + tcpdump: verbose output suppressed, use -v[v]... for full protocol decode + listening on cilium_wg0, link-type RAW (Raw IP), snapshot length 262144 bytes Welcome to nginx!

Welcome to nginx!

- ... - 40 packets captured 40 packets received by filter 0 packets dropped by kernel ``` +5. Deploy the Cilium connectivity resources to check and evaluate connectivity: + + ```sh + kubectl create ns cilium-test + kubectl apply -n cilium-test -f https://raw.githubusercontent.com/cilium/cilium/v1.14.1/examples/kubernetes/connectivity-check/connectivity-check.yaml + ``` + + ```text + deployment.apps/echo-a created + deployment.apps/echo-b created + deployment.apps/echo-b-host created + deployment.apps/pod-to-a created + deployment.apps/pod-to-external-1111 created + deployment.apps/pod-to-a-denied-cnp created + deployment.apps/pod-to-a-allowed-cnp created + deployment.apps/pod-to-external-fqdn-allow-google-cnp created + deployment.apps/pod-to-b-multi-node-clusterip created + deployment.apps/pod-to-b-multi-node-headless created + deployment.apps/host-to-b-multi-node-clusterip created + deployment.apps/host-to-b-multi-node-headless created + deployment.apps/pod-to-b-multi-node-nodeport created + deployment.apps/pod-to-b-intra-node-nodeport created + service/echo-a created + service/echo-b created + service/echo-b-headless created + service/echo-b-host-headless created + ciliumnetworkpolicy.cilium.io/pod-to-a-denied-cnp created + ciliumnetworkpolicy.cilium.io/pod-to-a-allowed-cnp created + ciliumnetworkpolicy.cilium.io/pod-to-external-fqdn-allow-google-cnp created + ``` + +6. View the logs of any of the connectivity tests to view the results: + + ```sh + kubectl logs echo-a-6575c98b7d-xknsv -n cilium-test + ``` + + ```text + \{^_^}/ hi! + + Loading /default.json + Done + + Resources + http://:8080/private + http://:8080/public + + Home + http://:8080 + + Type s + enter at any time to create a snapshot of the database + Watching... + + GET /public 200 7.063 ms - 57 + GET /public 200 3.126 ms - 57 + GET /public 200 3.039 ms - 57 + GET /public 200 2.776 ms - 57 + GET /public 200 3.087 ms - 57 + GET /public 200 2.781 ms - 57 + GET /public 200 2.977 ms - 57 + GET /public 200 2.596 ms - 57 + GET /public 200 2.991 ms - 57 + GET /public 200 2.708 ms - 57 + GET /public 200 3.066 ms - 57 + GET /public 200 2.616 ms - 57 + GET /public 200 2.875 ms - 57 + GET /public 200 2.689 ms - 57 + GET /public 200 2.800 ms - 57 + GET /public 200 2.556 ms - 57 + GET /public 200 3.238 ms - 57 + GET /public 200 2.538 ms - 57 + GET /public 200 2.890 ms - 57 + GET /public 200 2.666 ms - 57 + GET /public 200 2.729 ms - 57 + GET /public 200 2.580 ms - 57 + GET /public 200 2.919 ms - 57 + GET /public 200 2.630 ms - 57 + GET /public 200 2.857 ms - 57 + GET /public 200 2.716 ms - 57 + GET /public 200 1.693 ms - 57 + GET /public 200 2.715 ms - 57 + GET /public 200 2.729 ms - 57 + GET /public 200 2.655 ms - 57 + ``` + ## Destroy {% diff --git a/patterns/wireguard-with-cilium/eks.tf b/patterns/wireguard-with-cilium/eks.tf new file mode 100644 index 0000000000..5d448bcec3 --- /dev/null +++ b/patterns/wireguard-with-cilium/eks.tf @@ -0,0 +1,102 @@ +################################################################################ +# Cluster +################################################################################ + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.16" + + cluster_name = local.name + cluster_version = "1.27" + cluster_endpoint_public_access = true + + # EKS Addons + cluster_addons = { + coredns = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + initial = { + instance_types = ["m5.large"] + # Cilium Wireguard requires Linux Kernel 5.10 or above. + # For EKS 1.24 and above, the AMI the Kernel version is 5.10 + # For EKS 1.23 and below, you need to use Bottlerocket OS. For example: + # ami_type = "BOTTLEROCKET_x86_64" + # platform = "bottlerocket" + min_size = 1 + max_size = 3 + desired_size = 2 + } + } + # Extend node-to-node security group rules + node_security_group_additional_rules = { + # Cilium Wireguard Port https://github.com/cilium/cilium/blob/main/Documentation/security/network/encryption-wireguard.rst + ingress_cilium_wireguard = { + description = "Allow Cilium Wireguard node to node" + protocol = "udp" + from_port = 51871 + to_port = 51871 + type = "ingress" + self = true + } + } + + tags = local.tags +} + +################################################################################ +# Kubectl Output +################################################################################ + +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.7" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + helm_releases = { + cilium = { + name = "cilium" + chart = "cilium" + version = "1.14.1" + repository = "https://helm.cilium.io/" + description = "Cilium Add-on" + namespace = "kube-system" + create_namespace = false + + values = [ + <<-EOT + cni: + chainingMode: aws-cni + enableIPv4Masquerade: false + tunnel: disabled + endpointRoutes: + enabled: true + l7Proxy: false + encryption: + enabled: true + type: wireguard + EOT + ] + } + } + + tags = local.tags +} diff --git a/patterns/wireguard-with-cilium/main.tf b/patterns/wireguard-with-cilium/main.tf index 1b6f28e54c..e6c70ffea4 100644 --- a/patterns/wireguard-with-cilium/main.tf +++ b/patterns/wireguard-with-cilium/main.tf @@ -1,3 +1,29 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.47" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.9" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/wireguard-with-cilium/terraform.tfstate" + # } +} + provider "aws" { region = local.region } @@ -28,19 +54,9 @@ provider "helm" { } } -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} +################################################################################ +# Common data/locals +################################################################################ data "aws_availability_zones" "available" {} @@ -51,208 +67,12 @@ locals { vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" } } -################################################################################ -# Cluster -################################################################################ - -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 19.16" - - cluster_name = local.name - cluster_version = "1.27" - cluster_endpoint_public_access = true - - # EKS Addons - cluster_addons = { - coredns = {} - kube-proxy = {} - vpc-cni = {} - } - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - - eks_managed_node_groups = { - initial = { - instance_types = ["m5.large"] - # Cilium Wireguard requires Linux Kernel 5.10 or aboved. - # For EKS 1.24 and above, the AMI the Kernerl version is 5.10 - # For EKS 1.23 and below, you need to use Bottlerocket OS. For example: - # ami_type = "BOTTLEROCKET_x86_64" - # platform = "bottlerocket" - min_size = 1 - max_size = 3 - desired_size = 2 - } - } - # Extend node-to-node security group rules - node_security_group_additional_rules = { - ingress_cilium_wireguard = { - description = "Allow Cilium Wireguard node to node" - protocol = "udp" - from_port = 51871 - to_port = 51871 # Cilium Wireguard Port https://github.com/cilium/cilium/blob/main/Documentation/security/network/encryption-wireguard.rst - type = "ingress" - self = true - } - } - - tags = local.tags -} - -################################################################################ -# Cilium Helm Chart for e2e encryption with Wireguard -################################################################################ - -resource "helm_release" "cilium" { - name = "cilium" - chart = "cilium" - version = "1.13.2" - repository = "https://helm.cilium.io/" - description = "Cilium Add-on" - namespace = "kube-system" - create_namespace = false - - values = [ - <<-EOT - cni: - chainingMode: aws-cni - enableIPv4Masquerade: false - tunnel: disabled - endpointRoutes: - enabled: true - l7Proxy: false - encryption: - enabled: true - type: wireguard - EOT - ] - - depends_on = [ - module.eks - ] -} - -#--------------------------------------------------------------- -# Sample App for Testing -#--------------------------------------------------------------- - -# For some reason the example pods can't be deployed right after helm install of cilium a delay needs to be introduced. This is being investigated -resource "time_sleep" "wait_wireguard" { - count = var.enable_example ? 1 : 0 - create_duration = "15s" - - depends_on = [helm_release.cilium] -} - -resource "kubectl_manifest" "server" { - count = var.enable_example ? 1 : 0 - - yaml_body = yamlencode({ - apiVersion = "v1" - kind = "Pod" - metadata = { - name = "server" - labels = { - blog = "wireguard" - name = "server" - } - } - spec = { - containers = [ - { - name = "server" - image = "nginx" - } - ] - topologySpreadConstraints = [ - { - maxSkew = 1 - topologyKey = "kubernetes.io/hostname" - whenUnsatisfiable = "DoNotSchedule" - labelSelector = { - matchLabels = { - blog = "wireguard" - } - } - } - ] - } - }) - - depends_on = [time_sleep.wait_wireguard] -} - -resource "kubectl_manifest" "service" { - count = var.enable_example ? 1 : 0 - - yaml_body = yamlencode({ - apiVersion = "v1" - kind = "Service" - metadata = { - name = "server" - } - spec = { - selector = { - name = "server" - } - ports = [ - { - port = 80 - } - ] - } - }) -} - -resource "kubectl_manifest" "client" { - count = var.enable_example ? 1 : 0 - - yaml_body = yamlencode({ - apiVersion = "v1" - kind = "Pod" - metadata = { - name = "client" - labels = { - blog = "wireguard" - name = "client" - } - } - spec = { - containers = [ - { - name = "client" - image = "busybox" - command = ["watch", "wget", "server"] - } - ] - topologySpreadConstraints = [ - { - maxSkew = 1 - topologyKey = "kubernetes.io/hostname" - whenUnsatisfiable = "DoNotSchedule" - labelSelector = { - matchLabels = { - blog = "wireguard" - } - } - } - ] - } - }) - - depends_on = [kubectl_manifest.server] -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/patterns/wireguard-with-cilium/outputs.tf b/patterns/wireguard-with-cilium/outputs.tf deleted file mode 100644 index c624023e90..0000000000 --- a/patterns/wireguard-with-cilium/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "configure_kubectl" { - description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" - value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" -} diff --git a/patterns/wireguard-with-cilium/sample.tf b/patterns/wireguard-with-cilium/sample.tf new file mode 100644 index 0000000000..626a208013 --- /dev/null +++ b/patterns/wireguard-with-cilium/sample.tf @@ -0,0 +1,83 @@ +################################################################################ +# Sample Application +################################################################################ + +resource "kubernetes_pod_v1" "server" { + metadata { + name = "server" + labels = { + blog = "wireguard" + name = "server" + } + } + + spec { + container { + image = "nginx" + name = "server" + } + + topology_spread_constraint { + max_skew = 1 + topology_key = "kubernetes.io/hostname" + when_unsatisfiable = "DoNotSchedule" + + label_selector { + match_expressions { + key = "blog" + operator = "In" + values = ["wireguard"] + } + } + } + } +} + +resource "kubernetes_service_v1" "server" { + metadata { + name = "server" + } + spec { + selector = { + name = kubernetes_pod_v1.server.metadata[0].labels.name + } + + session_affinity = "ClientIP" + + port { + port = 80 + } + } +} + +resource "kubernetes_pod_v1" "client" { + metadata { + name = "client" + labels = { + blog = "wireguard" + name = "client" + } + } + + spec { + container { + image = "busybox" + name = "client" + command = ["watch", "wget", "server"] + } + + topology_spread_constraint { + max_skew = 1 + topology_key = "kubernetes.io/hostname" + when_unsatisfiable = "DoNotSchedule" + + label_selector { + match_expressions { + key = "blog" + operator = "In" + values = ["wireguard"] + } + } + } + } +} diff --git a/patterns/wireguard-with-cilium/variables.tf b/patterns/wireguard-with-cilium/variables.tf deleted file mode 100644 index 433e9ec9d1..0000000000 --- a/patterns/wireguard-with-cilium/variables.tf +++ /dev/null @@ -1,6 +0,0 @@ -# tflint-ignore: terraform_unused_declarations -variable "enable_example" { - description = "Enable example to test this blueprint" - type = bool - default = true -} diff --git a/patterns/wireguard-with-cilium/versions.tf b/patterns/wireguard-with-cilium/versions.tf deleted file mode 100644 index d42480c1e6..0000000000 --- a/patterns/wireguard-with-cilium/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.47" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.9" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.20" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.14" - } - time = { - source = "hashicorp/time" - version = ">= 0.9" - } - } - - # ## Used for end-to-end testing on project; update to suit your needs - # backend "s3" { - # bucket = "terraform-ssp-github-actions-state" - # region = "us-west-2" - # key = "e2e/wireguard-with-cilium/terraform.tfstate" - # } -} From 143ea9da75088d0247ed997fb17e3b30cf84c08a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 4 Oct 2023 15:29:38 -0400 Subject: [PATCH 2/2] chore: Update based on feedback --- .pre-commit-config.yaml | 2 +- patterns/wireguard-with-cilium/README.md | 52 ++++++++----- patterns/wireguard-with-cilium/eks.tf | 14 ++-- patterns/wireguard-with-cilium/example.yaml | 54 ++++++++++++++ patterns/wireguard-with-cilium/sample.tf | 83 --------------------- 5 files changed, 93 insertions(+), 112 deletions(-) create mode 100644 patterns/wireguard-with-cilium/example.yaml delete mode 100644 patterns/wireguard-with-cilium/sample.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07a682c97c..69734a100a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.2 + rev: v1.83.4 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/patterns/wireguard-with-cilium/README.md b/patterns/wireguard-with-cilium/README.md index d799203f41..0cfbeacb88 100644 --- a/patterns/wireguard-with-cilium/README.md +++ b/patterns/wireguard-with-cilium/README.md @@ -5,12 +5,11 @@ This pattern demonstrates Cilium configured in CNI chaining mode with the VPC CN - [Cilium CNI Chaining Documentation](https://docs.cilium.io/en/stable/installation/cni-chaining-aws-cni/) - [Cilium Wireguard Encryption Documentation](https://docs.cilium.io/en/stable/security/network/encryption-wireguard/) -## Areas of Interest +## Focal Points - `eks.tf` contains the cluster configuration and the deployment of Cilium. - - There are no specific requirements from an EKS perspective, other than the Linux Kernel version used by the OS must be 5.10+. - On Amazon EKS, this is available starting with EKS 1.24, or users can utilize the Bottlerocket OS for EKS < 1.23 -- `sample.tf` provides a sample application used to demonstrate the encrypted connectivity. This is optional and not required for the pattern. + - There are no specific requirements from an EKS perspective, other than the Linux Kernel version used by the OS must be 5.10+ +- `example.yaml` provides a sample application used to demonstrate the encrypted connectivity. This is optional and not required for the pattern. ## Deploy @@ -18,7 +17,19 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started ## Validate -1. Get the Cilium status from one of the Cilium pods. +1. Deploy the example pods: + + ```sh + kubectl apply -f example.yaml + ``` + + ```text + pod/server created + service/server created + pod/client created + ``` + +2. Get the Cilium status from one of the Cilium pods. Under the `Encryption` field, it should state `Wireguard` with a PubKey. `NodeEncryption: Disabled` is expected since `NodeEncryption` was not enabled @@ -31,42 +42,42 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started ```text Defaulted container "cilium-agent" out of: cilium-agent, config (init), mount-cgroup (init), apply-sysctl-overwrites (init), mount-bpf-fs (init), clean-cilium-state (init), install-cni-binaries (init) KVStore: Ok Disabled - Kubernetes: Ok 1.27+ (v1.27.4-eks-2d98532) [linux/amd64] + Kubernetes: Ok 1.28+ (v1.28.1-eks-43840fb) [linux/amd64] Kubernetes APIs: ["EndpointSliceOrEndpoint", "cilium/v2::CiliumClusterwideNetworkPolicy", "cilium/v2::CiliumEndpoint", "cilium/v2::CiliumNetworkPolicy", "cilium/v2::CiliumNode", "cilium/v2alpha1::CiliumCIDRGroup", "core/v1::Namespace", "core/v1::Pods", "core/v1::Service", "networking.k8s.io/v1::NetworkPolicy"] - KubeProxyReplacement: False [eth0 10.0.45.128 (Direct Routing), eth1 10.0.40.206] + KubeProxyReplacement: False [eth0 10.0.21.109 (Direct Routing), eth1 10.0.27.0] Host firewall: Disabled CNI Chaining: aws-cni - Cilium: Ok 1.14.1 (v1.14.1-c191ef6f) + Cilium: Ok 1.14.2 (v1.14.2-a6748946) NodeMonitor: Listening for events on 2 CPUs with 64x4096 of shared memory - Cilium health daemon: Ok - IPAM: IPv4: 1/254 allocated from 10.0.1.0/24, + Cilium health daemon: Ok + IPAM: IPv4: 1/254 allocated from 10.0.0.0/24, IPv4 BIG TCP: Disabled IPv6 BIG TCP: Disabled BandwidthManager: Disabled Host Routing: Legacy Masquerading: Disabled - Controller Status: 20/20 healthy + Controller Status: 24/24 healthy Proxy Status: No managed proxy redirect Global Identity Range: min 256, max 65535 - Hubble: Ok Current/Max Flows: 4095/4095 (100.00%), Flows/s: 1.58 Metrics: Disabled - Encryption: Wireguard [NodeEncryption: Disabled, cilium_wg0 (Pubkey: Es25c2idJtRzE0/FKAOvKPJ7ybRmZ23KrufK3HOuZTY=, Port: 51871, Peers: 1)] + Hubble: Ok Current/Max Flows: 410/4095 (10.01%), Flows/s: 1.59 Metrics: Disabled + Encryption: Wireguard [NodeEncryption: Disabled, cilium_wg0 (Pubkey: /yuqsZyG91AzVIkZ3AIq8qjQ0gGKQd6GWcRYh4LYpko=, Port: 51871, Peers: 1)] Cluster health: Probe disabled ``` -2. Open a shell inside the cilium container +3. Open a shell inside the cilium container ```sh kubectl -n kube-system exec -ti ds/cilium -- bash ``` -3. Install [`tcpdump`](https://www.tcpdump.org/) +4. Install [`tcpdump`](https://www.tcpdump.org/) ```sh apt-get update apt-get install -y tcpdump ``` -4. Start a packet capture on `cilium_wg0` and verify you see payload in clear text, it means the traffic is encrypted with wireguard +5. Start a packet capture on `cilium_wg0` and verify you see payload in clear text, it means the traffic is encrypted with wireguard ```sh tcpdump -A -c 40 -i cilium_wg0 | grep "Welcome to nginx!" @@ -82,7 +93,10 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started 0 packets dropped by kernel ``` -5. Deploy the Cilium connectivity resources to check and evaluate connectivity: + !!! info "Exit" + Exit the container shell by typing `exit` before continuing to next step + +6. Deploy the Cilium connectivity resources to check and evaluate connectivity: ```sh kubectl create ns cilium-test @@ -113,10 +127,10 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started ciliumnetworkpolicy.cilium.io/pod-to-external-fqdn-allow-google-cnp created ``` -6. View the logs of any of the connectivity tests to view the results: +7. View the logs of any of the connectivity tests to view the results: ```sh - kubectl logs echo-a-6575c98b7d-xknsv -n cilium-test + kubectl logs -n cilium-test ``` ```text diff --git a/patterns/wireguard-with-cilium/eks.tf b/patterns/wireguard-with-cilium/eks.tf index 5d448bcec3..f5b6c33009 100644 --- a/patterns/wireguard-with-cilium/eks.tf +++ b/patterns/wireguard-with-cilium/eks.tf @@ -7,7 +7,7 @@ module "eks" { version = "~> 19.16" cluster_name = local.name - cluster_version = "1.27" + cluster_version = "1.28" cluster_endpoint_public_access = true # EKS Addons @@ -23,16 +23,12 @@ module "eks" { eks_managed_node_groups = { initial = { instance_types = ["m5.large"] - # Cilium Wireguard requires Linux Kernel 5.10 or above. - # For EKS 1.24 and above, the AMI the Kernel version is 5.10 - # For EKS 1.23 and below, you need to use Bottlerocket OS. For example: - # ami_type = "BOTTLEROCKET_x86_64" - # platform = "bottlerocket" - min_size = 1 - max_size = 3 - desired_size = 2 + min_size = 1 + max_size = 3 + desired_size = 2 } } + # Extend node-to-node security group rules node_security_group_additional_rules = { # Cilium Wireguard Port https://github.com/cilium/cilium/blob/main/Documentation/security/network/encryption-wireguard.rst diff --git a/patterns/wireguard-with-cilium/example.yaml b/patterns/wireguard-with-cilium/example.yaml new file mode 100644 index 0000000000..77fb7c36b4 --- /dev/null +++ b/patterns/wireguard-with-cilium/example.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: server + labels: + blog: wireguard + name: server +spec: + containers: + - name: server + image: nginx + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + blog: wireguard +--- +apiVersion: v1 +kind: Service +metadata: + name: server +spec: + selector: + name: server + ports: + - port: 80 + sessionAffinity: ClientIP + sessionAffinityConfig: + clientIP: + timeoutSeconds: 600 +--- +apiVersion: v1 +kind: Pod +metadata: + name: client + labels: + blog: wireguard + name: client +spec: + containers: + - name: client + image: busybox + command: ["watch", "wget", "server"] + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + blog: wireguard +--- diff --git a/patterns/wireguard-with-cilium/sample.tf b/patterns/wireguard-with-cilium/sample.tf deleted file mode 100644 index 626a208013..0000000000 --- a/patterns/wireguard-with-cilium/sample.tf +++ /dev/null @@ -1,83 +0,0 @@ -################################################################################ -# Sample Application -################################################################################ - -resource "kubernetes_pod_v1" "server" { - metadata { - name = "server" - labels = { - blog = "wireguard" - name = "server" - } - } - - spec { - container { - image = "nginx" - name = "server" - } - - topology_spread_constraint { - max_skew = 1 - topology_key = "kubernetes.io/hostname" - when_unsatisfiable = "DoNotSchedule" - - label_selector { - match_expressions { - key = "blog" - operator = "In" - values = ["wireguard"] - } - } - } - } -} - -resource "kubernetes_service_v1" "server" { - metadata { - name = "server" - } - spec { - selector = { - name = kubernetes_pod_v1.server.metadata[0].labels.name - } - - session_affinity = "ClientIP" - - port { - port = 80 - } - } -} - -resource "kubernetes_pod_v1" "client" { - metadata { - name = "client" - labels = { - blog = "wireguard" - name = "client" - } - } - - spec { - container { - image = "busybox" - name = "client" - command = ["watch", "wget", "server"] - } - - topology_spread_constraint { - max_skew = 1 - topology_key = "kubernetes.io/hostname" - when_unsatisfiable = "DoNotSchedule" - - label_selector { - match_expressions { - key = "blog" - operator = "In" - values = ["wireguard"] - } - } - } - } -}