Fix: collector failing to scrape targets (#174)

* Move external labels to resource attributes * Avoid DDoS with using 0.0.0.0 * Pre-commit
aws-observability · Jun 7, 2023 · 3979910 · 3979910
1 parent 9b6b234
commit 3979910
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 7 deletions.
diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md
@@ -113,7 +113,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this
 | <a name="input_tags"></a> [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no |
 | <a name="input_target_secret_name"></a> [target\_secret\_name](#input\_target\_secret\_name) | Target secret in Kubernetes to store the Grafana API Key Secret | `string` | `"grafana-admin-credentials"` | no |
 | <a name="input_target_secret_namespace"></a> [target\_secret\_namespace](#input\_target\_secret\_namespace) | Target namespace of secret in Kubernetes to store the Grafana API Key Secret | `string` | `"grafana-operator"` | no |
-| <a name="input_tracing_config"></a> [tracing\_config](#input\_tracing\_config) | Configuration object for traces collection to AWS X-Ray | <pre>object({<br>    otlp_grpc_endpoint = string<br>    otlp_http_endpoint = string<br>    send_batch_size    = number<br>    timeout            = string<br>  })</pre> | <pre>{<br>  "otlp_grpc_endpoint": "0.0.0.0:4317",<br>  "otlp_http_endpoint": "0.0.0.0:4318",<br>  "send_batch_size": 50,<br>  "timeout": "30s"<br>}</pre> | no |
+| <a name="input_tracing_config"></a> [tracing\_config](#input\_tracing\_config) | Configuration object for traces collection to AWS X-Ray | <pre>object({<br>    otlp_grpc_endpoint = string<br>    otlp_http_endpoint = string<br>    send_batch_size    = number<br>    timeout            = string<br>  })</pre> | <pre>{<br>  "otlp_grpc_endpoint": "localhost:4317",<br>  "otlp_http_endpoint": "localhost:4318",<br>  "send_batch_size": 50,<br>  "timeout": "30s"<br>}</pre> | no |
 
 ## Outputs
 

diff --git a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml
@@ -36,9 +36,6 @@ spec:
           global:
             scrape_interval: {{ .Values.globalScrapeInterval }}
             scrape_timeout: {{ .Values.globalScrapeTimeout }}
-            external_labels:
-              cluster: {{ .Values.ekscluster }}
-              region: {{ .Values.region }}
           scrape_configs:
             - job_name: 'kubernetes-kubelet'
               scrape_interval: {{ .Values.globalScrapeInterval }}
@@ -1775,6 +1772,8 @@ spec:
         endpoint: {{ .Values.ampurl }}
         auth:
           authenticator: sigv4auth
+        resource_to_telemetry_conversion:
+          enabled: true
       logging:
         loglevel: warn
     extensions:
@@ -1790,6 +1789,17 @@ spec:
       batch/metrics:
         timeout: 30s
         send_batch_size: 500
+      attributes/metrics:
+        actions:
+          - key: cluster
+            action: upsert
+            value: {{ .Values.ekscluster }}
+          - key: region
+            action: upsert
+            value: {{ .Values.region }}
+          - key: account_id
+            action: upsert
+            value: {{ .Values.accountId }}
     {{ if .Values.enableTracing }}
       batch/traces:
         timeout: {{ .Values.tracingTimeout }}
@@ -1800,7 +1810,7 @@ spec:
       pipelines:
         metrics:
           receivers: [prometheus, otlp]
-          processors: [batch/metrics]
+          processors: [batch/metrics, attributes/metrics]
           exporters: [logging, prometheusremotewrite]
         {{ if .Values.enableTracing }}
         traces:

diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf
@@ -210,8 +210,8 @@ variable "tracing_config" {
   })
 
   default = {
-    otlp_grpc_endpoint = "0.0.0.0:4317"
-    otlp_http_endpoint = "0.0.0.0:4318"
+    otlp_grpc_endpoint = "localhost:4317"
+    otlp_http_endpoint = "localhost:4318"
     send_batch_size    = 50
     timeout            = "30s"
   }