From 5b56c948870123a1dde12ba31d71fb489ff61fc9 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Tue, 19 Sep 2023 20:40:26 -0400 Subject: [PATCH 1/7] Adding `patterns/containerinsights` --- .../fluentbit-containerinsights/README.md | 68 +++++ patterns/fluentbit-containerinsights/main.tf | 268 ++++++++++++++++++ .../fluentbit-containerinsights/outputs.tf | 4 + .../fluentbit-containerinsights/variables.tf | 0 .../fluentbit-containerinsights/versions.tf | 18 ++ 5 files changed, 358 insertions(+) create mode 100644 patterns/fluentbit-containerinsights/README.md create mode 100644 patterns/fluentbit-containerinsights/main.tf create mode 100644 patterns/fluentbit-containerinsights/outputs.tf create mode 100644 patterns/fluentbit-containerinsights/variables.tf create mode 100644 patterns/fluentbit-containerinsights/versions.tf diff --git a/patterns/fluentbit-containerinsights/README.md b/patterns/fluentbit-containerinsights/README.md new file mode 100644 index 0000000000..d66ec2e4b9 --- /dev/null +++ b/patterns/fluentbit-containerinsights/README.md @@ -0,0 +1,68 @@ +# Amazon EKS Blueprints Addons - Complete + +Configuration in this directory creates: + +- An EKS cluster (required to support module/tests) +- A number of addons (see code for full list) deployed via Terraform + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which will incur monetary charges on your AWS bill. Run `terraform destroy` when you no longer need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 4.47 | +| [helm](#requirement\_helm) | >= 2.8 | +| [kubernetes](#requirement\_kubernetes) | >= 2.20 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 4.47 | +| [aws.virginia](#provider\_aws.virginia) | >= 4.47 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [adot\_irsa](#module\_adot\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 | +| [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 | +| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.13 | +| [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | ../../ | n/a | +| [velero\_backup\_s3\_bucket](#module\_velero\_backup\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_security_group.guardduty](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_vpc_endpoint.guardduty](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [configure\_kubectl](#output\_configure\_kubectl) | Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig | + + +Apache-2.0 Licensed. See [LICENSE](https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/main/LICENSE) diff --git a/patterns/fluentbit-containerinsights/main.tf b/patterns/fluentbit-containerinsights/main.tf new file mode 100644 index 0000000000..53de861387 --- /dev/null +++ b/patterns/fluentbit-containerinsights/main.tf @@ -0,0 +1,268 @@ +provider "aws" { + region = local.region +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } + } +} + +data "aws_availability_zones" "available" {} + +locals { + name = basename(path.cwd) + region = "us-west-2" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } +} + +################################################################################ +# Cluster +################################################################################ + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.13" + + cluster_name = local.name + cluster_version = "1.26" + cluster_endpoint_public_access = true + + cluster_addons = { + coredns = {} + kube-proxy = {} + vpc-cni = {} + aws-guardduty-agent = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + cluster_enabled_log_types = ["api", "audit", "authenticator", "controllerManager", "scheduler"] + + manage_aws_auth_configmap = true + + eks_managed_node_groups = { + initial = { + instance_types = ["m5.xlarge"] + + min_size = 2 + max_size = 10 + desired_size = 3 + } + } + + tags = local.tags +} + +################################################################################ +# Blueprints Addons +################################################################################ + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + enable_metrics_server = true + enable_aws_for_fluentbit = true + aws_for_fluentbit_cw_log_group = { + create = true + use_name_prefix = true # Set this to true to enable name prefix + name_prefix = "eks-cluster-logs-" + retention = 7 + } + aws_for_fluentbit = { + enable_containerinsights = true + set = [{ + name = "cloudWatchLogs.autoCreateGroup" + value = true + }, + { + name = "hostNetwork" + value = true + }, + { + name = "dnsPolicy" + value = "ClusterFirstWithHostNet" + } + ] + s3_bucket_arns = [ + module.logs_s3_bucket.s3_bucket_arn, + "${module.logs_s3_bucket.s3_bucket_arn}/logs/*" + ] + } + + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 10)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} + +resource "aws_security_group" "guardduty" { + name = "guardduty_vpce_allow_tls" + description = "Allow TLS inbound traffic" + vpc_id = module.vpc.vpc_id + + ingress { + description = "TLS from VPC" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = local.tags +} + +resource "aws_vpc_endpoint" "guardduty" { + vpc_id = module.vpc.vpc_id + service_name = "com.amazonaws.${local.region}.guardduty-data" + subnet_ids = module.vpc.private_subnets + vpc_endpoint_type = "Interface" + security_group_ids = [aws_security_group.guardduty.id] + private_dns_enabled = true + + tags = local.tags +} + +module "logs_s3_bucket" { + source = "terraform-aws-modules/s3-bucket/aws" + version = "~> 3.0" + + bucket_prefix = "${local.name}-" + + # Allow deletion of non-empty bucket + # NOTE: This is enabled for example usage only, you should not enable this for production workloads + force_destroy = true + + attach_deny_insecure_transport_policy = true + attach_require_latest_tls_policy = true + + acl = "private" + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true + + control_object_ownership = true + object_ownership = "BucketOwnerPreferred" + + versioning = { + status = true + mfa_delete = false + } + + server_side_encryption_configuration = { + rule = { + apply_server_side_encryption_by_default = { + sse_algorithm = "AES256" + } + } + } + + tags = local.tags +} + +# ################################################################################ +# # AWS For FluentBut Kubelet Monitoring with ContainerInsights +# ################################################################################ +# # For why this is required: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/ContainerInsights-use-kubelet.html + +# resource "kubernetes_cluster_role_v1" "aws-for-fluentbit" { + +# metadata { +# name = "aws-for-fluentbit-role" +# } +# rule { +# non_resource_urls = ["/metrics"] +# verbs = ["get"] +# } +# rule { +# api_groups = [""] +# resources = ["namespaces", "pods", "pods/logs", "nodes", "nodes/proxy"] +# verbs = ["get", "list", "watch"] +# } +# } + +# resource "kubernetes_role_binding_v1" "aws-for-fluentbit" { +# metadata { +# name = "aws-for-fluentbit-rolebinding" +# namespace = "kube-system" +# } +# role_ref { +# api_group = "rbac.authorization.k8s.io" +# kind = "ClusterRole" +# name = kubernetes_cluster_role_v1.aws-for-fluentbit.metadata[0].name +# } +# subject { +# kind = "ServiceAccount" +# name = "aws-for-fluent-bit-sa" +# namespace = "kube-system" +# } +# } diff --git a/patterns/fluentbit-containerinsights/outputs.tf b/patterns/fluentbit-containerinsights/outputs.tf new file mode 100644 index 0000000000..c624023e90 --- /dev/null +++ b/patterns/fluentbit-containerinsights/outputs.tf @@ -0,0 +1,4 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" +} diff --git a/patterns/fluentbit-containerinsights/variables.tf b/patterns/fluentbit-containerinsights/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/patterns/fluentbit-containerinsights/versions.tf b/patterns/fluentbit-containerinsights/versions.tf new file mode 100644 index 0000000000..824e91ecae --- /dev/null +++ b/patterns/fluentbit-containerinsights/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.47" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.8" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20" + } + } +} From b7195bfe2bbed59a3ad08378b29b11609317d90e Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 20 Sep 2023 12:33:02 -0400 Subject: [PATCH 2/7] Adding `patterns/fluentbit-containerinsights` --- .../fluentbit-containerinsights/README.md | 359 +++++++++++++++--- 1 file changed, 312 insertions(+), 47 deletions(-) diff --git a/patterns/fluentbit-containerinsights/README.md b/patterns/fluentbit-containerinsights/README.md index d66ec2e4b9..22d9202a24 100644 --- a/patterns/fluentbit-containerinsights/README.md +++ b/patterns/fluentbit-containerinsights/README.md @@ -1,68 +1,333 @@ -# Amazon EKS Blueprints Addons - Complete +# AWS for FluentBit with ContainerInsights and Kubelet monitoring. -Configuration in this directory creates: +This pattern demonstrates an Amazon EKS Cluster deployment using AWS for FluentBit integration with ContainerInsights and Kubelet monitoring to send logs to Amazon CloudWatch Logs. -- An EKS cluster (required to support module/tests) -- A number of addons (see code for full list) deployed via Terraform +- [Container Insights on Amazon EKS and Kubernetes](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Container-Insights-setup-EKS-quickstart.html#Container-Insights-setup-EKS-quickstart-FluentBit) +- [Use_Kubelet feature](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/ContainerInsights-use-kubelet.html) -## Usage +## Deploy -To run this example you need to execute: +See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and steps to deploy this pattern. + +## Validate + +1. List all the Pods in the cluster; you should see `aws-for-fluentbit-` Pods running on `kube-system` Namespace. All the Pods should reach a status of `Running` after approximately 60 seconds: + +terraform output configure_kubectl | jq -r | bash + +```bash +$ kubectl get pods -A +``` +```bash +NAMESPACE NAME READY STATUS RESTARTS AGE +amazon-guardduty aws-guardduty-agent-2bkb8 1/1 Running 0 4m28s +amazon-guardduty aws-guardduty-agent-mdj2q 1/1 Running 0 4m28s +amazon-guardduty aws-guardduty-agent-vxkvg 1/1 Running 0 4m28s +kube-system aws-for-fluent-bit-54lmh 1/1 Running 0 4m34s +kube-system aws-for-fluent-bit-89wbv 1/1 Running 0 4m33s +kube-system aws-for-fluent-bit-nt2qr 1/1 Running 0 4m34s +kube-system aws-node-q55f2 1/1 Running 0 4m25s +kube-system aws-node-tx5s8 1/1 Running 0 4m8s +kube-system aws-node-vlgpr 1/1 Running 0 4m16s +kube-system coredns-6c45d94f67-dvmvf 1/1 Running 0 8m51s +kube-system coredns-6c45d94f67-xfb97 1/1 Running 0 8m51s +kube-system kube-proxy-5qcdm 1/1 Running 0 4m42s +kube-system kube-proxy-9hksp 1/1 Running 0 4m41s +kube-system kube-proxy-c2hcx 1/1 Running 0 4m42s +kube-system metrics-server-8794b9cdf-l7hzq 1/1 Running 0 6m8s +``` + +2. Validate the `aws-for-fluent-bit` configMap was created correctly in the `kube-system` Namespace. Make sure that the configMap has the `application-log.conf`, `dataplane-log.conf`, `host-log.conf`, and `fluent-bit.conf` definitions. Also, check the parameters `Use_Kubelet On` and `Kubelet_Port 10250` are set! ```bash -$ terraform init -$ terraform plan -$ terraform apply +$ kubectl get cm -n kube-system aws-for-fluent-bit -o yaml ``` +```yaml +apiVersion: v1 +data: + application-log.conf: | + [INPUT] + Name tail + Tag application.* + Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy* + Path /var/log/containers/*.log + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_container.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + Read_from_Head Off + + [INPUT] + Name tail + Tag application.* + Path /var/log/containers/fluent-bit* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_log.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head Off + + [INPUT] + Name tail + Tag application.* + Path /var/log/containers/cloudwatch-agent* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_cwagent.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head Off -Note that this example may create resources which will incur monetary charges on your AWS bill. Run `terraform destroy` when you no longer need these resources. + [FILTER] + Name kubernetes + Match application.* + Kube_URL https://kubernetes.default.svc:443 + Kube_Tag_Prefix application.var.log.containers. + Merge_Log On + Merge_Log_Key log_processed + K8S-Logging.Parser On + K8S-Logging.Exclude Off + Labels Off + Annotations Off + Use_Kubelet On + Kubelet_Port 10250 + Buffer_Size 0 - -## Requirements + [OUTPUT] + Name cloudwatch_logs + Match application.* + region us-west-2 + log_group_name /aws/containerinsights/fluentbit-containerinsights/application + log_stream_prefix ${HOSTNAME}- + auto_create_group true + extra_user_agent container-insights + workers 1 + dataplane-log.conf: | + [INPUT] + Name systemd + Tag dataplane.systemd.* + Systemd_Filter _SYSTEMD_UNIT=docker.service + Systemd_Filter _SYSTEMD_UNIT=containerd.service + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + DB /var/fluent-bit/state/systemd.db + Path /var/log/journal + Read_From_Tail On -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 4.47 | -| [helm](#requirement\_helm) | >= 2.8 | -| [kubernetes](#requirement\_kubernetes) | >= 2.20 | + [INPUT] + Name tail + Tag dataplane.tail.* + Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_dataplane_tail.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + Read_from_Head Off -## Providers + [FILTER] + Name modify + Match dataplane.systemd.* + Rename _HOSTNAME hostname + Rename _SYSTEMD_UNIT systemd_unit + Rename MESSAGE message + Remove_regex ^((?!hostname|systemd_unit|message).)*$ -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 4.47 | -| [aws.virginia](#provider\_aws.virginia) | >= 4.47 | + [FILTER] + Name aws + Match dataplane.* + imds_version v2 -## Modules + [OUTPUT] + Name cloudwatch_logs + Match dataplane.* + region us-west-2 + log_group_name /aws/containerinsights/fluentbit-containerinsights/dataplane + log_stream_prefix ${HOSTNAME}- + auto_create_group true + extra_user_agent container-insights + fluent-bit.conf: | + [SERVICE] + Flush 5 + Grace 30 + Log_Level info + Daemon off + Parsers_File parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + storage.path /var/fluent-bit/state/flb-storage/ + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 5M -| Name | Source | Version | -|------|--------|---------| -| [adot\_irsa](#module\_adot\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 | -| [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 | -| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.13 | -| [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | ../../ | n/a | -| [velero\_backup\_s3\_bucket](#module\_velero\_backup\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | + @INCLUDE application-log.conf + @INCLUDE dataplane-log.conf + @INCLUDE host-log.conf + host-log.conf: | + [INPUT] + Name tail + Tag host.dmesg + Path /var/log/dmesg + Key message + DB /var/fluent-bit/state/flb_dmesg.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head Off -## Resources + [INPUT] + Name tail + Tag host.messages + Path /var/log/messages + Parser syslog + DB /var/fluent-bit/state/flb_messages.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head Off -| Name | Type | -|------|------| -| [aws_security_group.guardduty](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_vpc_endpoint.guardduty](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | + [INPUT] + Name tail + Tag host.secure + Path /var/log/secure + Parser syslog + DB /var/fluent-bit/state/flb_secure.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head Off -## Inputs + [FILTER] + Name aws + Match host.* + imds_version v2 -No inputs. + [OUTPUT] + Name cloudwatch_logs + Match host.* + region us-west-2 + log_group_name /aws/containerinsights/fluentbit-containerinsights/host + log_stream_prefix ${HOSTNAME}. + auto_create_group true + extra_user_agent container-insights + parsers.conf: | + [PARSER] + Name syslog + Format regex + Regex ^(?