tempo(traces): profiles enabled

Signed-off-by: Weifeng Wang <[email protected]> clean dashboards Signed-off-by: Weifeng Wang <[email protected]> agent tracing enabled Signed-off-by: Weifeng Wang <[email protected]> tempo(traces): profiles enabled Signed-off-by: Weifeng Wang <[email protected]>
qclaogui · Mar 4, 2024 · 341cdf2 · 341cdf2
1 parent 7a4d508
commit 341cdf2
Show file tree

Hide file tree

Showing 35 changed files with 1,309 additions and 8,062 deletions.
diff --git a/Makefile b/Makefile
@@ -314,9 +314,12 @@ delete-monolithic-mode-profiles:
 deploy-monolithic-mode-traces: deploy-grafana ## Deploy monolithic-mode traces
 	$(info ******************** deploy monolithic-mode traces manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/traces | kubectl apply -f -
+	@$(KUSTOMIZE) build monitoring-mixins | kubectl apply -f -
 	kubectl rollout status -n tracing-system statefulset/tempo --watch --timeout=600s
 	@kubectl rollout restart deployment -n gateway nginx
 	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@kubectl rollout restart daemonset -n monitoring-system grafana-agent
+	kubectl rollout status -n monitoring-system daemonset/grafana-agent --watch --timeout=600s
 	@kubectl rollout restart deployment -n monitoring-system grafana
 	kubectl rollout status -n monitoring-system deployment/grafana --watch --timeout=600s
 	@echo ""
@@ -334,6 +337,8 @@ deploy-monolithic-mode-all-in-one: deploy-grafana ## Deploy monolithic-mode all-
 	kubectl rollout status -n monitoring-system deployment/mimir --watch --timeout=600s
 	@kubectl rollout restart deployment -n gateway nginx
 	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@kubectl rollout restart daemonset -n monitoring-system grafana-agent
+	kubectl rollout status -n monitoring-system daemonset/grafana-agent --watch --timeout=600s	
 	@kubectl rollout restart deployment -n monitoring-system grafana
 	kubectl rollout status -n monitoring-system deployment/grafana --watch --timeout=600s
 	@echo ""
@@ -424,9 +429,12 @@ delete-microservices-mode-profiles:
 deploy-microservices-mode-traces: deploy-grafana ## Deploy microservices-mode traces
 	$(info ******************** deploy microservices-mode traces manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/traces | kubectl apply -f -
+	@$(KUSTOMIZE) build monitoring-mixins | kubectl apply -f -
 	kubectl rollout status -n tracing-system statefulset/tempo-distributed-ingester --watch --timeout=600s
 	@kubectl rollout restart deployment -n gateway nginx
 	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@kubectl rollout restart daemonset -n monitoring-system grafana-agent
+	kubectl rollout status -n monitoring-system daemonset/grafana-agent --watch --timeout=600s
 	@kubectl rollout restart deployment -n monitoring-system grafana
 	kubectl rollout status -n monitoring-system deployment/grafana --watch --timeout=600s
 	@echo ""

diff --git a/docker-compose/common/config/agent-flow/monolithic-mode-all.river b/docker-compose/common/config/agent-flow/monolithic-mode-all.river
@@ -150,7 +150,7 @@ loki.process "containers" {
 	}
 
 	stage.tenant {
-		value = "fake"
+		value = "anonymous"
 	}
 
 	forward_to = [module.file.docker_compose.exports.logs_receiver]

diff --git a/docker-compose/common/config/agent-flow/traces.river b/docker-compose/common/config/agent-flow/traces.river
@@ -188,6 +188,10 @@ otelcol.receiver.otlp "containers" {
 }
 
 otelcol.processor.batch "containers" {
+	send_batch_size     = 16384
+	send_batch_max_size = 0
+	timeout             = "2s"
+
 	output {
 		metrics = [otelcol.processor.memory_limiter.containers.input]
 		logs    = [otelcol.processor.memory_limiter.containers.input]
@@ -224,7 +228,7 @@ loki.process "containers" {
 	}
 
 	stage.tenant {
-		value = "fake"
+		value = "anonymous"
 	}
 
 	forward_to = [module.file.docker_compose.exports.logs_receiver]

diff --git a/docker-compose/microservices-mode/traces/docker-compose.yaml b/docker-compose/microservices-mode/traces/docker-compose.yaml
@@ -158,14 +158,14 @@ services:
       - ../../common/config/grafana/provisioning:/etc/grafana/provisioning
       - ../../common/config/tempo/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml # load datasources
       - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin
-      - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      # - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
       - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin
       - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin
     environment:
       - GF_LOG_LEVEL=error
       - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password}
       - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles
       - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS=agent:4317
-      - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=namespace:monitoring-system
+      - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=service.namespace:monitoring-system
     ports:
       - "3000:3000"
diff --git a/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml b/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml
@@ -186,7 +186,7 @@ services:
       - ./config/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
       - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin
       - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
-      - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      # - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
       - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin
       - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin
     environment:

diff --git a/docker-compose/monolithic-mode/traces/docker-compose.yaml b/docker-compose/monolithic-mode/traces/docker-compose.yaml
@@ -113,14 +113,14 @@ services:
       - ../../common/config/grafana/provisioning:/etc/grafana/provisioning
       - ../../common/config/tempo/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml # load datasources
       - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin
-      - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      # - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
       - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin
       - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin
     environment:
       - GF_LOG_LEVEL=warn
       - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password}
       - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles
       - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS=agent:4317
-      - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=namespace:monitoring-system
+      - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=service.namespace:monitoring-system
     ports:
       - "3000:3000"
diff --git a/kubernetes/common/grafana-agent/configs/modules/traces.river b/kubernetes/common/grafana-agent/configs/modules/traces.river
@@ -181,7 +181,7 @@ loki.process "traceslogs" {
 	}
 
 	stage.tenant {
-		value = "fake"
+		value = "anonymous"
 	}
 
 	forward_to = argument.logs_forward_to.value

diff --git a/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml b/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml
@@ -613,11 +613,11 @@ data:
     \"tracesmetrics\" {\n\tforward_to = argument.metrics_forward_to.value\n}\n\notelcol.exporter.loki
     \"traceslogs\" {\n\tforward_to = [loki.process.traceslogs.receiver]\n}\n\nloki.process
     \"traceslogs\" {\n\tstage.drop {\n\t\tlonger_than = \"8KB\"\n\t\tolder_than  =
-    \"12h\"\n\t}\n\n\tstage.tenant {\n\t\tvalue = \"fake\"\n\t}\n\n\tforward_to =
-    argument.logs_forward_to.value\n}\n"
+    \"12h\"\n\t}\n\n\tstage.tenant {\n\t\tvalue = \"anonymous\"\n\t}\n\n\tforward_to
+    = argument.logs_forward_to.value\n}\n"
 kind: ConfigMap
 metadata:
-  name: agent-modules-d2t5hbhhgg
+  name: agent-modules-86g2bf77m7
   namespace: monitoring-system
 ---
 apiVersion: v1
@@ -839,7 +839,7 @@ spec:
           path: /var/log
         name: varlog
       - configMap:
-          name: agent-modules-d2t5hbhhgg
+          name: agent-modules-86g2bf77m7
         name: agent-modules
 ---
 apiVersion: monitoring.coreos.com/v1

diff --git a/kubernetes/microservices-mode/traces/configs/config.river b/kubernetes/microservices-mode/traces/configs/config.river
@@ -34,27 +34,52 @@ module.file "traces_primary" {
 	}
 }
 
-/********************************************
- * Metrics
- ********************************************/
-module.file "metrics_primary" {
-	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/metrics.river"
+tracing {
+	sampling_fraction = 0.8
+	write_to          = [otelcol.processor.batch.containers.input]
+}
 
-	arguments {
-		forward_to = [module.file.lgtmp.exports.metrics_receiver]
-		clustering = true
+otelcol.processor.batch "containers" {
+	send_batch_size     = 16384
+	send_batch_max_size = 0
+	timeout             = "2s"
+
+	output {
+		metrics = [otelcol.processor.memory_limiter.containers.input]
+		traces  = [otelcol.processor.memory_limiter.containers.input]
 	}
 }
 
+otelcol.processor.memory_limiter "containers" {
+	check_interval         = "1s"
+	limit_percentage       = 50
+	spike_limit_percentage = 30
+
+	output {
+		metrics = [otelcol.processor.k8sattributes.containers.input]
+		traces  = [otelcol.processor.k8sattributes.containers.input]
+	}
+}
+
+otelcol.processor.k8sattributes "containers" {
+	output {
+		metrics = [otelcol.exporter.prometheus.containers.input]
+		traces  = [module.file.lgtmp.exports.traces_receiver]
+	}
+}
+
+otelcol.exporter.prometheus "containers" {
+	forward_to = [module.file.lgtmp.exports.metrics_receiver]
+}
+
 /********************************************
- * Agent Integrations
+ * Metrics
  ********************************************/
-module.file "agent_integrations" {
-	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/integrations.river"
+module.file "metrics_primary" {
+	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/metrics.river"
 
 	arguments {
-		name       = "agent-integrations"
-		namespace  = "monitoring-system"
 		forward_to = [module.file.lgtmp.exports.metrics_receiver]
+		clustering = true
 	}
 }
diff --git a/kubernetes/microservices-mode/traces/configs/tempo-distributed.yaml b/kubernetes/microservices-mode/traces/configs/tempo-distributed.yaml
@@ -64,7 +64,7 @@ metrics_generator:
       - http.status_code
       - service.version
   storage:
-    path: /tmp/tempo/generator/wal
+    path: /var/tempo/generator/wal
     remote_write_add_org_id_header: true
     remote_write:
     - url: http://nginx.gateway.svc.cluster.local:8080/api/v1/push