diff --git a/docs/patterns/images/ADOT_container_logs.png b/docs/patterns/images/ADOT_container_logs.png new file mode 100644 index 00000000..d05106b0 Binary files /dev/null and b/docs/patterns/images/ADOT_container_logs.png differ diff --git a/docs/patterns/images/ADOT_container_logs_group.png b/docs/patterns/images/ADOT_container_logs_group.png new file mode 100644 index 00000000..160e9bdc Binary files /dev/null and b/docs/patterns/images/ADOT_container_logs_group.png differ diff --git a/docs/patterns/images/ADOT_container_logs_insights.png b/docs/patterns/images/ADOT_container_logs_insights.png new file mode 100644 index 00000000..5c95dc04 Binary files /dev/null and b/docs/patterns/images/ADOT_container_logs_insights.png differ diff --git a/docs/patterns/images/ADOT_container_logs_insights_results.png b/docs/patterns/images/ADOT_container_logs_insights_results.png new file mode 100644 index 00000000..6ee5c917 Binary files /dev/null and b/docs/patterns/images/ADOT_container_logs_insights_results.png differ diff --git a/docs/patterns/single-new-eks-observability-accelerators/single-new-eks-container-logs-opensource-observability.md b/docs/patterns/single-new-eks-observability-accelerators/single-new-eks-container-logs-opensource-observability.md new file mode 100644 index 00000000..7084bb7d --- /dev/null +++ b/docs/patterns/single-new-eks-observability-accelerators/single-new-eks-container-logs-opensource-observability.md @@ -0,0 +1,84 @@ +# Single Cluster Open Source Observability - Container Logs Collection + +## Objective + +Following the [announcement](https://aws.amazon.com/about-aws/whats-new/2023/11/logs-support-aws-distro-opentelemetry/) of logs support in AWS Distro for OpenTelemetry, this pattern demonstrates how to use the _New EKS Cluster Open Source Observability Accelerator_ to forward container logs to cloud watch using ADOT containers log collector. + +## Prerequisites + +Ensure that you have installed the following tools on your machine. + +1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) +3. [cdk](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install) +4. [npm](https://docs.npmjs.com/cli/v8/commands/npm-install) + +## Deploying + +Please follow the _Deploying_ instructions of the [New EKS Cluster Open Source Observability Accelerator](./single-new-eks-opensource-observability.md) pattern, except for step 7, where you need to replace "context" in `~/.cdk.json` with the following: + +```typescript + "context": { + "fluxRepository": { + "name": "grafana-dashboards", + "namespace": "grafana-operator", + "repository": { + "repoUrl": "https://github.com/aws-observability/aws-observability-accelerator", + "name": "grafana-dashboards", + "targetRevision": "main", + "path": "./artifacts/grafana-operator-manifests/eks/infrastructure" + }, + "values": { + "GRAFANA_CLUSTER_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/cluster.json", + "GRAFANA_KUBELET_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/kubelet.json", + "GRAFANA_NSWRKLDS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/namespace-workloads.json", + "GRAFANA_NODEEXP_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodeexporter-nodes.json", + "GRAFANA_NODES_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodes.json", + "GRAFANA_WORKLOADS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/workloads.json" + }, + "kustomizations": [ + { + "kustomizationPath": "./artifacts/grafana-operator-manifests/eks/infrastructure" + } + ] + }, + "adotcontainerlogs.pattern.enabled": true + } +``` + +!! warning This scenario might need larger worker node for the pod. + + +Once completed the rest of the _Deploying_ steps, you can move on with the deployment of the Nginx workload. + +## Viewing Logs in CloudWatch Log Groups and Logs Insights + +Navigate to CloudWatch, then go to "Log groups" + +Search for log group with the name "/aws/eks/single-new-eks-mixed-observability-accelerator" and open it + +You will see log streams created using the node name + +![ADOT_container_logs_group](../images/ADOT_container_logs_group.png) + +Open the log stream and you view the logs forwarded by the container logs collector to CloudWatch + +![ADOT_container_logs](../images/ADOT_container_logs.png) + +Navigate to CloudWatch, then go to "Logs Insights" + +In the dropdown, select log group with name "/aws/eks/single-new-eks-mixed-observability-accelerator" and run a query. + +![ADOT_container_logs_insights](../images/ADOT_container_logs_insights.png) + +Then you can view the results of your query: + +![ADOT_container_logs_insights](../images/ADOT_container_logs_insights_results.png) + +## Teardown + +You can teardown the whole CDK stack with the following command: + +```bash +make pattern single-new-eks-opensource-observability destroy +``` diff --git a/lib/common/resources/otel-collector-config.yml b/lib/common/resources/otel-collector-config.yml index 30eeb6c3..14b76d04 100644 --- a/lib/common/resources/otel-collector-config.yml +++ b/lib/common/resources/otel-collector-config.yml @@ -9,7 +9,7 @@ metadata: namespace: "{{namespace}}" spec: mode: "{{deploymentMode}}" - image: public.ecr.aws/aws-observability/aws-otel-collector:v0.33.1 + image: public.ecr.aws/aws-observability/aws-otel-collector:v0.37.0 resources: limits: cpu: "1" @@ -18,6 +18,22 @@ spec: cpu: "1" memory: "2Gi" serviceAccount: adot-collector + podSecurityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - name: varlogpods + mountPath: /var/log/pods + readOnly: true + volumes: + - name: varlogpods + hostPath: + path: /var/log/pods + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName config: | receivers: prometheus: @@ -1740,6 +1756,86 @@ spec: source_labels: - __meta_kubernetes_pod_phase {{ stop enableIstioMonJob }} + {{ start enableAdotContainerLogsReceiver }} + filelog: + include: [ /var/log/pods/*/*/*.log ] + include_file_name: false + include_file_path: true + start_at: end + operators: + # Find out which format is used by kubernetes + - type: router + id: get-format + routes: + - output: parser-docker + expr: 'body matches "^\\{"' + - output: parser-crio + expr: 'body matches "^[^ Z]+ "' + - output: parser-containerd + expr: 'body matches "^[^ Z]+Z"' + # Parse CRI-O format + - type: regex_parser + id: parser-crio + regex: + '^(?P