diff --git a/docs/eks/logs.md b/docs/eks/logs.md index 7cc33f95..bddc0f92 100644 --- a/docs/eks/logs.md +++ b/docs/eks/logs.md @@ -20,18 +20,19 @@ to enable Amazon CloudWatch as a data source. Make sure to provide permissions. Amazon CloudWatch data source has already been setup for you. All logs are delivered in the following CloudWatch Log groups naming pattern: -`/aws/eks/observability-accelerator/{cluster-name}/{namespace}`. Log streams -follow `{container-name}.{pod-name}`. In Grafana, querying and analyzing logs +`/aws/eks/observability-accelerator/{cluster-name}/workloads`. Log streams +follow the naming pattern `{node-name}`. In Grafana, querying and analyzing logs is done with [CloudWatch Logs Insights](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/AnalyzingLogData.html) ### Example - ADOT collector logs -Select one or many log groups and run the following query. The example below, -queries AWS Distro for OpenTelemetry (ADOT) logs +Select workloads log group for the cluster and run the following query. The example below, +queries container logs from `kube-system` namespace. ```console -fields @timestamp, log -| order @timestamp desc +fields @timestamp, @message, @logStream, @log, resource.k8s.namespace.name +| filter resource.k8s.namespace.name = "kube-system" +| sort @timestamp desc | limit 100 ``` @@ -49,8 +50,9 @@ In the example below, we use the following query to graph the number of metrics collected by the ADOT collector ```console -fields @timestamp, log -| parse log /"#metrics": (?\d+)}/ +fields @timestamp, attributes.log +| filter resource.k8s.namespace.name = "adot-collector-kubeprometheus" +| parse attributes.log /\"metrics\": (?\d+?)(,|\})/ | stats avg(metrics_count) by bin(5m) | limit 100 ``` diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 2899667e..fa043e78 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -3,7 +3,7 @@ This module provides EKS cluster monitoring with the following resources: - AWS Distro For OpenTelemetry Operator and Collector for Metrics and Traces -- Logs with [AWS for FluentBit](https://github.com/aws/aws-for-fluent-bit) +- Logs with [ADOT Container Logs Collector](https://aws-otel.github.io/docs/getting-started/adot-eks-add-on/config-container-logs) - Installs Grafana Operator to add AWS data sources and create Grafana Dashboards to Amazon Managed Grafana. - Installs FluxCD to perform GitOps sync of a Git Repo to EKS Cluster. We will use this later for creating Grafana Dashboards and AWS datasources to Amazon Managed Grafana. - Installs External Secrets Operator to retrieve and Sync the Grafana API keys from AWS SSM Parameter Store. @@ -37,8 +37,8 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | Name | Source | Version | |------|--------|---------| +| [adot\_logs](#module\_adot\_logs) | ./add-ons/adot-logs | n/a | | [external\_secrets](#module\_external\_secrets) | ./add-ons/external-secrets | n/a | -| [fluentbit\_logs](#module\_fluentbit\_logs) | ./add-ons/aws-for-fluentbit | n/a | | [helm\_addon](#module\_helm\_addon) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.32.1 | | [istio\_monitoring](#module\_istio\_monitoring) | ./patterns/istio | n/a | | [java\_monitoring](#module\_java\_monitoring) | ./patterns/java | n/a | @@ -90,7 +90,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [enable\_istio](#input\_enable\_istio) | Enable ISTIO workloads monitoring, alerting and default dashboards | `bool` | `false` | no | | [enable\_java](#input\_enable\_java) | Enable Java workloads monitoring, alerting and default dashboards | `bool` | `false` | no | | [enable\_kube\_state\_metrics](#input\_enable\_kube\_state\_metrics) | Enables or disables Kube State metrics exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no | -| [enable\_logs](#input\_enable\_logs) | Using AWS For FluentBit to collect cluster and application logs to Amazon CloudWatch | `bool` | `true` | no | +| [enable\_logs](#input\_enable\_logs) | Using ADOT container logs collector to collect cluster and application logs to Amazon CloudWatch | `bool` | `true` | no | | [enable\_managed\_prometheus](#input\_enable\_managed\_prometheus) | Creates a new Amazon Managed Service for Prometheus Workspace | `bool` | `true` | no | | [enable\_nginx](#input\_enable\_nginx) | Enable NGINX workloads monitoring, alerting and default dashboards | `bool` | `false` | no | | [enable\_node\_exporter](#input\_enable\_node\_exporter) | Enables or disables Node exporter. Disabling this might affect some data in the dashboards | `bool` | `true` | no | diff --git a/modules/eks-monitoring/add-ons/adot-logs/README.md b/modules/eks-monitoring/add-ons/adot-logs/README.md new file mode 100644 index 00000000..5d6d9611 --- /dev/null +++ b/modules/eks-monitoring/add-ons/adot-logs/README.md @@ -0,0 +1,53 @@ +# AWS Distro for OpenTelemetry (ADOT) Container Logs Collector + +[AWS Distro for OpenTelemetry (ADOT)](https://aws-otel.github.io/) is a secure, +production-ready, AWS-supported distribution of the OpenTelemetry project. +Part of the Cloud Native Computing Foundation, OpenTelemetry provides open +source APIs, libraries, and agents to collect distributed traces and metrics +for application monitoring. + +This module generates the +[ADOT Container Logs Collector](https://aws-otel.github.io/docs/getting-started/adot-eks-add-on/config-container-logs) configuration for Amazon EKS ADOT add-on. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1.0 | +| [aws](#requirement\_aws) | >= 3.72 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.72 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [adot\_logs\_iam\_role](#module\_adot\_logs\_iam\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | 5.33.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.adot_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_policy.adot_logs_iam_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy_document.adot_logs_iam_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [addon\_config](#input\_addon\_config) | ADOT Container Logs Collector config |
object({
enable_logs = bool
logs_config = object({
cw_log_retention_days = number
})
})
|
{
"enable_logs": true,
"logs_config": {
"cw_log_retention_days": 90
}
}
| no | +| [addon\_context](#input\_addon\_context) | Input configuration for the addon |
object({
aws_caller_identity_account_id = string
aws_caller_identity_arn = string
aws_eks_cluster_endpoint = string
aws_partition_id = string
aws_region_name = string
eks_cluster_id = string
eks_oidc_issuer_url = string
eks_oidc_provider_arn = string
irsa_iam_role_path = string
irsa_iam_permissions_boundary = string
tags = map(string)
})
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [adot\_logs\_collector\_config](#output\_adot\_logs\_collector\_config) | ADOT Container Logs Collector configuration | + diff --git a/modules/eks-monitoring/add-ons/adot-logs/data.tf b/modules/eks-monitoring/add-ons/adot-logs/data.tf new file mode 100644 index 00000000..ec95b681 --- /dev/null +++ b/modules/eks-monitoring/add-ons/adot-logs/data.tf @@ -0,0 +1,33 @@ +data "aws_iam_policy_document" "adot_logs_iam_policy" { + statement { + sid = "PutLogEvents" + effect = "Allow" + resources = ["arn:${var.addon_context.aws_partition_id}:logs:${var.addon_context.aws_region_name}:${var.addon_context.aws_caller_identity_account_id}:log-group:/aws/eks/observability-accelerator/${var.addon_context.eks_cluster_id}/workloads:log-stream:*"] + actions = ["logs:PutLogEvents"] + } + + statement { + sid = "DescribeLogGroups" + effect = "Allow" + resources = ["*"] + + actions = [ + "logs:DescribeLogGroups", + ] + } + + statement { + sid = "LogStreams" + effect = "Allow" + resources = [ + "arn:${var.addon_context.aws_partition_id}:logs:${var.addon_context.aws_region_name}:${var.addon_context.aws_caller_identity_account_id}:log-group:/aws/eks/observability-accelerator/${var.addon_context.eks_cluster_id}/workloads", + "arn:${var.addon_context.aws_partition_id}:logs:${var.addon_context.aws_region_name}:${var.addon_context.aws_caller_identity_account_id}:log-group:/aws/eks/observability-accelerator/${var.addon_context.eks_cluster_id}/workloads:log-stream:*" + ] + + actions = [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + ] + } + +} diff --git a/modules/eks-monitoring/add-ons/adot-logs/main.tf b/modules/eks-monitoring/add-ons/adot-logs/main.tf new file mode 100644 index 00000000..e32046ef --- /dev/null +++ b/modules/eks-monitoring/add-ons/adot-logs/main.tf @@ -0,0 +1,42 @@ +resource "aws_cloudwatch_log_group" "adot_log_group" { + count = var.addon_config.enable_logs ? 1 : 0 + + name = "/aws/eks/observability-accelerator/${var.addon_context.eks_cluster_id}/workloads" + + retention_in_days = var.addon_config.logs_config.cw_log_retention_days + + tags = var.addon_context.tags +} + +resource "aws_iam_policy" "adot_logs_iam_policy" { + count = var.addon_config.enable_logs ? 1 : 0 + + name = "${substr(var.addon_context.eks_cluster_id, 0, 30)}-${var.addon_context.aws_region_name}-adot-logs-policy" + path = "/" + description = "IAM Policy for ADOT Container Logs Collector" + + policy = data.aws_iam_policy_document.adot_logs_iam_policy.json + tags = var.addon_context.tags +} + +module "adot_logs_iam_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.33.0" + + count = var.addon_config.enable_logs ? 1 : 0 + + role_name = "${substr(var.addon_context.eks_cluster_id, 0, 30)}-${var.addon_context.aws_region_name}-adot-logs-irsa" + + role_policy_arns = { + policy = resource.aws_iam_policy.adot_logs_iam_policy[0].arn + } + + oidc_providers = { + main = { + provider_arn = var.addon_context.eks_oidc_provider_arn + namespace_service_accounts = ["opentelemetry-operator-system:adot-col-container-logs"] + } + } + + tags = var.addon_context.tags +} diff --git a/modules/eks-monitoring/add-ons/adot-logs/outputs.tf b/modules/eks-monitoring/add-ons/adot-logs/outputs.tf new file mode 100644 index 00000000..ce2faed3 --- /dev/null +++ b/modules/eks-monitoring/add-ons/adot-logs/outputs.tf @@ -0,0 +1,37 @@ +output "adot_logs_collector_config" { + description = "ADOT Container Logs Collector configuration" + value = jsondecode(length(resource.aws_cloudwatch_log_group.adot_log_group) > 0 ? jsonencode({ + resources = { + limits = { + cpu = "1000m" + memory = "750Mi" + } + + requests = { + cpu = "300m" + memory = "512Mi" + } + } + + serviceAccount = { + annotations = { + "eks.amazonaws.com/role-arn" = module.adot_logs_iam_role[0].iam_role_arn + } + } + + exporters = { + awscloudwatchlogs = { + log_group_name = "/aws/eks/observability-accelerator/$CLUSTER_NAME/workloads" + log_stream_name = "$NODE_NAME" + } + } + + pipelines = { + logs = { + cloudwatchLogs = { + enabled = true + } + } + } + }) : "{}") +} diff --git a/modules/eks-monitoring/add-ons/adot-logs/variables.tf b/modules/eks-monitoring/add-ons/adot-logs/variables.tf new file mode 100644 index 00000000..b852902d --- /dev/null +++ b/modules/eks-monitoring/add-ons/adot-logs/variables.tf @@ -0,0 +1,32 @@ +variable "addon_context" { + description = "Input configuration for the addon" + type = object({ + aws_caller_identity_account_id = string + aws_caller_identity_arn = string + aws_eks_cluster_endpoint = string + aws_partition_id = string + aws_region_name = string + eks_cluster_id = string + eks_oidc_issuer_url = string + eks_oidc_provider_arn = string + irsa_iam_role_path = string + irsa_iam_permissions_boundary = string + tags = map(string) + }) +} + +variable "addon_config" { + description = "ADOT Container Logs Collector config" + type = object({ + enable_logs = bool + logs_config = object({ + cw_log_retention_days = number + }) + }) + default = { + enable_logs = true + logs_config = { + cw_log_retention_days = 90 + } + } +} diff --git a/modules/eks-monitoring/add-ons/adot-logs/versions.tf b/modules/eks-monitoring/add-ons/adot-logs/versions.tf new file mode 100644 index 00000000..e257f04e --- /dev/null +++ b/modules/eks-monitoring/add-ons/adot-logs/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + } +} diff --git a/modules/eks-monitoring/add-ons/adot-operator/main.tf b/modules/eks-monitoring/add-ons/adot-operator/main.tf index 3a68306b..ee34eca2 100644 --- a/modules/eks-monitoring/add-ons/adot-operator/main.tf +++ b/modules/eks-monitoring/add-ons/adot-operator/main.tf @@ -34,6 +34,8 @@ resource "aws_eks_addon" "adot" { service_account_role_arn = try(var.addon_config.service_account_role_arn, null) preserve = try(var.addon_config.preserve, true) + configuration_values = try(var.addon_config.configuration_values, null) + tags = merge( var.addon_context.tags, try(var.addon_config.tags, {}), diff --git a/modules/eks-monitoring/add-ons/aws-for-fluentbit/README.md b/modules/eks-monitoring/add-ons/aws-for-fluentbit/README.md deleted file mode 100644 index 8b37ec6a..00000000 --- a/modules/eks-monitoring/add-ons/aws-for-fluentbit/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# AWS for Fluent Bit - -Fluent Bit is an open source Log Processor and Forwarder which allows you to collect any data like metrics and logs from different sources, enrich them with filters and send them to multiple destinations. -AWS provides a Fluent Bit image with plugins for CloudWatch Logs, Kinesis Data Firehose, Kinesis Data Stream and Amazon OpenSearch Service. - -This add-on is configured to stream the worker node logs to CloudWatch Logs by default. It can be configured to stream the logs to additional destinations like Kinesis Data Firehose, Kinesis Data Streams and Amazon OpenSearch Service by passing the custom `values.yaml`. -See this [Helm Chart](https://github.com/aws/eks-charts/tree/master/stable/aws-for-fluent-bit) for more details. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [aws](#requirement\_aws) | >= 3.72 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.72 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [helm\_addon](#module\_helm\_addon) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.32.1 | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_policy.aws_for_fluent_bit](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy_document.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [addon\_context](#input\_addon\_context) | Input configuration for the addon |
object({
aws_caller_identity_account_id = string
aws_caller_identity_arn = string
aws_eks_cluster_endpoint = string
aws_partition_id = string
aws_region_name = string
eks_cluster_id = string
eks_oidc_issuer_url = string
eks_oidc_provider_arn = string
tags = map(string)
irsa_iam_role_path = string
irsa_iam_permissions_boundary = string
})
| n/a | yes | -| [cw\_log\_retention\_days](#input\_cw\_log\_retention\_days) | FluentBit CloudWatch Log group retention period | `number` | `90` | no | -| [helm\_config](#input\_helm\_config) | Helm provider config aws\_for\_fluent\_bit. | `any` | `{}` | no | -| [irsa\_policies](#input\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no | -| [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps. | `bool` | `false` | no | -| [refresh\_interval](#input\_refresh\_interval) | FluentBit input refresh interval | `number` | `60` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [irsa\_arn](#output\_irsa\_arn) | IAM role ARN for the service account | -| [irsa\_name](#output\_irsa\_name) | IAM role name for the service account | -| [release\_metadata](#output\_release\_metadata) | Map of attributes of the Helm release metadata | -| [service\_account](#output\_service\_account) | Name of Kubernetes service account | - diff --git a/modules/eks-monitoring/add-ons/aws-for-fluentbit/data.tf b/modules/eks-monitoring/add-ons/aws-for-fluentbit/data.tf deleted file mode 100644 index 270723df..00000000 --- a/modules/eks-monitoring/add-ons/aws-for-fluentbit/data.tf +++ /dev/null @@ -1,23 +0,0 @@ -data "aws_iam_policy_document" "irsa" { - statement { - sid = "PutLogEvents" - effect = "Allow" - resources = ["arn:${var.addon_context.aws_partition_id}:logs:${var.addon_context.aws_region_name}:${var.addon_context.aws_caller_identity_account_id}:log-group:*:log-stream:*"] - actions = ["logs:PutLogEvents"] - } - - statement { - sid = "CreateCWLogs" - effect = "Allow" - resources = ["arn:${var.addon_context.aws_partition_id}:logs:${var.addon_context.aws_region_name}:${var.addon_context.aws_caller_identity_account_id}:log-group:*"] - - actions = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:PutRetentionPolicy", - ] - } - -} diff --git a/modules/eks-monitoring/add-ons/aws-for-fluentbit/locals.tf b/modules/eks-monitoring/add-ons/aws-for-fluentbit/locals.tf deleted file mode 100644 index e77d609e..00000000 --- a/modules/eks-monitoring/add-ons/aws-for-fluentbit/locals.tf +++ /dev/null @@ -1,48 +0,0 @@ -locals { - name = "aws-for-fluent-bit" - service_account = try(var.helm_config.service_account, "${local.name}-sa") - - set_values = [ - { - name = "serviceAccount.name" - value = local.service_account - }, - { - name = "serviceAccount.create" - value = false - } - ] - - # https://github.com/aws/eks-charts/blob/master/stable/aws-for-fluent-bit/Chart.yaml - default_helm_config = { - name = local.name - chart = local.name - repository = "https://aws.github.io/eks-charts" - version = "0.1.27" - namespace = local.name - values = local.default_helm_values - description = "aws-for-fluentbit Helm Chart deployment configuration" - } - - helm_config = merge( - local.default_helm_config, - var.helm_config - ) - - default_helm_values = [templatefile("${path.module}/values.yaml", { - aws_region = var.addon_context.aws_region_name - cluster_name = var.addon_context.eks_cluster_id - log_retention_days = var.cw_log_retention_days - refresh_interval = var.refresh_interval - service_account = local.service_account - })] - - irsa_config = { - kubernetes_namespace = local.helm_config["namespace"] - kubernetes_service_account = local.service_account - create_kubernetes_namespace = try(local.helm_config["create_namespace"], true) - create_kubernetes_service_account = true - create_service_account_secret_token = try(local.helm_config["create_service_account_secret_token"], false) - irsa_iam_policies = concat([aws_iam_policy.aws_for_fluent_bit.arn], var.irsa_policies) - } -} diff --git a/modules/eks-monitoring/add-ons/aws-for-fluentbit/main.tf b/modules/eks-monitoring/add-ons/aws-for-fluentbit/main.tf deleted file mode 100644 index 126d04a6..00000000 --- a/modules/eks-monitoring/add-ons/aws-for-fluentbit/main.tf +++ /dev/null @@ -1,15 +0,0 @@ -module "helm_addon" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.32.1" - manage_via_gitops = var.manage_via_gitops - set_values = local.set_values - helm_config = local.helm_config - irsa_config = local.irsa_config - addon_context = var.addon_context -} - -resource "aws_iam_policy" "aws_for_fluent_bit" { - name = "${var.addon_context.eks_cluster_id}-fluentbit" - description = "IAM Policy for AWS for FluentBit" - policy = data.aws_iam_policy_document.irsa.json - tags = var.addon_context.tags -} diff --git a/modules/eks-monitoring/add-ons/aws-for-fluentbit/outputs.tf b/modules/eks-monitoring/add-ons/aws-for-fluentbit/outputs.tf deleted file mode 100644 index 37b305f9..00000000 --- a/modules/eks-monitoring/add-ons/aws-for-fluentbit/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "release_metadata" { - description = "Map of attributes of the Helm release metadata" - value = module.helm_addon.release_metadata -} - -output "irsa_arn" { - description = "IAM role ARN for the service account" - value = module.helm_addon.irsa_arn -} - -output "irsa_name" { - description = "IAM role name for the service account" - value = module.helm_addon.irsa_name -} - -output "service_account" { - description = "Name of Kubernetes service account" - value = module.helm_addon.service_account -} diff --git a/modules/eks-monitoring/add-ons/aws-for-fluentbit/values.yaml b/modules/eks-monitoring/add-ons/aws-for-fluentbit/values.yaml deleted file mode 100644 index be4afb9f..00000000 --- a/modules/eks-monitoring/add-ons/aws-for-fluentbit/values.yaml +++ /dev/null @@ -1,30 +0,0 @@ -serviceAccount: - create: false - name: ${service_account} - -cloudWatch: - enabled: false - -cloudWatchLogs: - enabled: true - region: ${aws_region} - # logGroupName is a fallback to failed parsing - logGroupName: /aws/eks/observability-accelerator/workloads - logGroupTemplate: /aws/eks/observability-accelerator/${cluster_name}/$kubernetes['namespace_name'] - logStreamTemplate: $kubernetes['container_name'].$kubernetes['pod_name'] - logKey: log - logRetentionDays: ${log_retention_days} - -input: - enabled: false - -additionalInputs: | - [INPUT] - Name tail - Tag kube.* - Path /var/log/containers/*.log - DB /var/log/flb_kube.db - Mem_Buf_Limit 5MB - Skip_Long_Lines On - Refresh_Interval ${refresh_interval} - multiline.parser cri, docker, go, java, python diff --git a/modules/eks-monitoring/add-ons/aws-for-fluentbit/variables.tf b/modules/eks-monitoring/add-ons/aws-for-fluentbit/variables.tf deleted file mode 100644 index cbab80a1..00000000 --- a/modules/eks-monitoring/add-ons/aws-for-fluentbit/variables.tf +++ /dev/null @@ -1,47 +0,0 @@ -variable "helm_config" { - description = "Helm provider config aws_for_fluent_bit." - type = any - default = {} -} - -variable "cw_log_retention_days" { - description = "FluentBit CloudWatch Log group retention period" - type = number - default = 90 -} - -variable "refresh_interval" { - description = "FluentBit input refresh interval" - type = number - default = 60 -} - - -variable "manage_via_gitops" { - type = bool - description = "Determines if the add-on should be managed via GitOps." - default = false -} - -variable "irsa_policies" { - description = "Additional IAM policies for a IAM role for service accounts" - type = list(string) - default = [] -} - -variable "addon_context" { - description = "Input configuration for the addon" - type = object({ - aws_caller_identity_account_id = string - aws_caller_identity_arn = string - aws_eks_cluster_endpoint = string - aws_partition_id = string - aws_region_name = string - eks_cluster_id = string - eks_oidc_issuer_url = string - eks_oidc_provider_arn = string - tags = map(string) - irsa_iam_role_path = string - irsa_iam_permissions_boundary = string - }) -} diff --git a/modules/eks-monitoring/add-ons/aws-for-fluentbit/versions.tf b/modules/eks-monitoring/add-ons/aws-for-fluentbit/versions.tf deleted file mode 100644 index f92f41b9..00000000 --- a/modules/eks-monitoring/add-ons/aws-for-fluentbit/versions.tf +++ /dev/null @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.0.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.72" - } - } -} diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index c96525ee..22dbd6f7 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -5,6 +5,16 @@ resource "aws_prometheus_workspace" "this" { tags = var.tags } +module "adot_logs" { + source = "./add-ons/adot-logs" + + addon_context = local.context + addon_config = { + enable_logs = var.enable_logs + logs_config = var.logs_config + } +} + module "operator" { source = "./add-ons/adot-operator" count = var.enable_amazon_eks_adot ? 1 : 0 @@ -12,6 +22,13 @@ module "operator" { enable_cert_manager = var.enable_cert_manager kubernetes_version = local.eks_cluster_version addon_context = local.context + addon_config = { + configuration_values = jsonencode({ + collector = { + containerLogs = module.adot_logs.adot_logs_collector_config + } + }) + } } resource "helm_release" "kube_state_metrics" { @@ -249,14 +266,6 @@ module "istio_monitoring" { pattern_config = coalesce(var.istio_config, local.istio_pattern_config) } -module "fluentbit_logs" { - source = "./add-ons/aws-for-fluentbit" - count = var.enable_logs ? 1 : 0 - - cw_log_retention_days = var.logs_config.cw_log_retention_days - addon_context = local.context -} - module "external_secrets" { source = "./add-ons/external-secrets" count = var.enable_external_secrets ? 1 : 0 diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf index 34e7532c..0f97eeeb 100644 --- a/modules/eks-monitoring/variables.tf +++ b/modules/eks-monitoring/variables.tf @@ -397,7 +397,7 @@ variable "istio_config" { } variable "enable_logs" { - description = "Using AWS For FluentBit to collect cluster and application logs to Amazon CloudWatch" + description = "Using ADOT container logs collector to collect cluster and application logs to Amazon CloudWatch" type = bool default = true } @@ -409,7 +409,7 @@ variable "logs_config" { }) default = { - # Valid values are [1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653] + # Valid values are [1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653] cw_log_retention_days = 90 } }