Merge pull request #120 from qclaogui:meta-monitoring

[example] Enabled local LGTMP Stack meta monitoring
qclaogui · Apr 27, 2024 · 94abc3e · 94abc3e
2 parents 95fab62 + fecda55
commit 94abc3e
Show file tree

Hide file tree

Showing 17 changed files with 496 additions and 368 deletions.
diff --git a/alloy-modules/compose/logs/keep-labels.alloy b/alloy-modules/compose/logs/keep-labels.alloy
@@ -19,16 +19,16 @@ declare "keep_labels" {
 		comment  = "List of labels to keep before the log message is written to Loki"
 		default  = [
 			"app",
+			"job",
+			"region",
 			"cluster",
 			"namespace",
 			"pod",
-			"instance",
 			"container",
 			"component",
+			"instance",
 			"env",
-			"job",
 			"level",
-			"region",
 			"service",
 			"squad",
 			"team",

diff --git a/alloy-modules/compose/logs/labels-scrape.alloy b/alloy-modules/compose/logs/labels-scrape.alloy
@@ -20,11 +20,6 @@ declare "labels_scrape" {
 		comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
 	}
 
-	argument "job_label" {
-		optional = true
-		default  = "integrations/docker/cadvisor"
-	}
-
 	argument "instance_name" {
 		optional = true
 	}
@@ -104,18 +99,33 @@ declare "labels_scrape" {
 			regex = "^(" + argument.tenant.value + ")$"
 		}
 
+		// set a default job label to be the namespace/service_name
 		rule {
 			action        = "replace"
-			source_labels = ["__meta_docker_container_id"]
-			replacement   = argument.job_label.value
-			target_label  = "job"
+			source_labels = [
+				"__meta_docker_container_label_com_docker_compose_service",
+			]
+			regex        = "^(?:;*)?([^;]+).*$"
+			replacement  = argument.namespace.value + "/$1"
+			target_label = "job"
 		}
 
+		// allow resources to declare their the job label value to use when collecting their logs, the default value is "",
 		rule {
 			action        = "replace"
-			source_labels = ["__meta_docker_container_id"]
-			replacement   = coalesce(argument.instance_name.value, constants.hostname)
-			target_label  = "instance"
+			source_labels = [
+				"__meta_docker_container_label_" + argument.__sd_label.value + "_job",
+			]
+			separator    = ";"
+			regex        = "^(?:;*)?([^;]+).*$"
+			replacement  = "$1"
+			target_label = "job"
+		}
+
+		rule {
+			source_labels = ["__meta_docker_container_name"]
+			regex         = "/(.*)"
+			target_label  = "pod"
 		}
 
 		rule {

diff --git a/alloy-modules/compose/metrics/labels-scrape.alloy b/alloy-modules/compose/metrics/labels-scrape.alloy
@@ -333,13 +333,9 @@ declare "labels_scrape" {
 		}
 
 		rule {
-			action        = "replace"
-			source_labels = [
-				"__meta_docker_container_label_com_docker_compose_service",
-			]
-			regex        = "^(?:;*)?([^;]+).*$"
-			replacement  = "$1"
-			target_label = "pod"
+			source_labels = ["__meta_docker_container_name"]
+			regex         = "/(.*)"
+			target_label  = "pod"
 		}
 
 		rule {

diff --git a/alloy-modules/compose/profiles/labels-scrape.alloy b/alloy-modules/compose/profiles/labels-scrape.alloy
@@ -202,13 +202,9 @@ declare "labels_scrape" {
 		}
 
 		rule {
-			action        = "replace"
-			source_labels = [
-				"__meta_docker_container_label_com_docker_compose_service",
-			]
-			regex        = "^(?:;*)?([^;]+).*$"
-			replacement  = "$1"
-			target_label = "pod"
+			source_labels = ["__meta_docker_container_name"]
+			regex         = "/(.*)"
+			target_label  = "pod"
 		}
 
 		rule {

diff --git a/examples/meta-monitoring/README.md b/examples/meta-monitoring/README.md
@@ -0,0 +1,70 @@
+# LGTMP Meta Monitoring
+
+This example enabled local LGTMP Stack meta monitoring by setting `loki`,`tempo`, `mimir` and `pyroscope` 's lable `metrics.grafana.com/scrape: true`(default: false).
+
+## Docker compose
+
+`compose.yaml`:
+
+```yaml
+include:
+# # use git remote
+# - path: https://github.com/qclaogui/codelab-monitoring.git#main:docker-compose/monolithic-mode/all-in-one/compose.yaml
+
+# # use local path
+- path: ../../docker-compose/monolithic-mode/all-in-one/compose.yaml
+
+```
+
+`compose.override.yaml`:
+
+```yaml
+services:
+  loki:
+    labels:
+      metrics.grafana.com/scrape: true
+  tempo:
+    labels:
+      metrics.grafana.com/scrape: true
+  mimir:
+    labels:
+      metrics.grafana.com/scrape: true
+  pyroscope:
+    labels:
+      metrics.grafana.com/scrape: true
+
+  # override included service grafana for dashboards pre-provisioning
+  grafana:
+    volumes:
+      - ../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
+      - ../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      - ../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin
+      - ../../monitoring-mixins/pyroscope-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/pyroscope-mixin
+
+  # override included service mimirtool for rules and alerts pre-provisioning
+  mimirtool:
+    volumes:
+      - ../../monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/rules/loki-mixin-rules.yaml
+      - ../../monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/rules/loki-mixin-alerts.yaml
+      - ../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
+      - ../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
+      - ../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-rules.yaml:/rules/mimir-mixin-rules.yaml
+      - ../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-alerts.yaml:/rules/mimir-mixin-alerts.yaml
+      - ../../monitoring-mixins/pyroscope-mixin/deploy/pyroscope-mixin-rules.yaml:/rules/pyroscope-mixin-rules.yaml
+```
+
+## Try it
+
+```shell
+COMPOSE_EXPERIMENTAL_GIT_REMOTE=true docker compose up -d --remove-orphans
+```
+
+Once all containers are up and running you can search for metrics in Grafana.
+
+Navigate to [http://localhost:3000](http://localhost:3000)
+
+## Stop services
+
+```shell
+COMPOSE_EXPERIMENTAL_GIT_REMOTE=true docker compose down
+```
diff --git a/examples/meta-monitoring/compose.override.yaml b/examples/meta-monitoring/compose.override.yaml
@@ -0,0 +1,35 @@
+
+services:
+  loki:
+    labels:
+      metrics.grafana.com/scrape: true
+      metrics.grafana.com/job: "monitoring-system/loki-single-binary"
+      logs.grafana.com/job: "monitoring-system/loki-single-binary"
+  tempo:
+    labels:
+      metrics.grafana.com/scrape: true
+  mimir:
+    labels:
+      metrics.grafana.com/scrape: true
+  pyroscope:
+    labels:
+      metrics.grafana.com/scrape: true
+
+  # override included service grafana for dashboards pre-provisioning
+  grafana:
+    volumes:
+      - ../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
+      - ../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      - ../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin
+      - ../../monitoring-mixins/pyroscope-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/pyroscope-mixin
+
+  # override included service mimirtool for rules and alerts pre-provisioning
+  mimirtool:
+    volumes:
+      - ../../monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/rules/loki-mixin-rules.yaml
+      - ../../monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/rules/loki-mixin-alerts.yaml
+      - ../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
+      - ../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
+      - ../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-rules.yaml:/rules/mimir-mixin-rules.yaml
+      - ../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-alerts.yaml:/rules/mimir-mixin-alerts.yaml
+      - ../../monitoring-mixins/pyroscope-mixin/deploy/pyroscope-mixin-rules.yaml:/rules/pyroscope-mixin-rules.yaml
diff --git a/examples/meta-monitoring/compose.yaml b/examples/meta-monitoring/compose.yaml
@@ -0,0 +1,17 @@
+# ============================================================================ #
+#                           Meta Monitoring                                   
+# ============================================================================ #
+
+# Usage:
+#   COMPOSE_EXPERIMENTAL_GIT_REMOTE=true docker compose up -d --remove-orphans
+#   COMPOSE_EXPERIMENTAL_GIT_REMOTE=true docker compose down
+
+# Note:
+# include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later.
+# docs: https://docs.docker.com/compose/multiple-compose-files/include/#include-and-overrides
+include:
+# # use git remote
+# - path: https://github.com/qclaogui/codelab-monitoring.git#main:docker-compose/monolithic-mode/all-in-one/compose.yaml
+
+# # use local path
+- path: ../../docker-compose/monolithic-mode/all-in-one/compose.yaml
diff --git a/monitoring-mixins/loki-mixin/deploy.libsonnet b/monitoring-mixins/loki-mixin/deploy.libsonnet
@@ -2,9 +2,13 @@
   // Config overrides
   _config+:: {
 
+    // uses Grafana Alloy instead of promtail
     promtail+: {
       enabled: false,
     },
 
+    meta_monitoring+: {
+      enabled: true,
+    },
   },
 }