Merge pull request #58 from qclaogui:add-option-to-collect-metrics

Add option to collect metrics

docker-compose/common/compose-include/agent-collect-all.yaml

@@ -1,6 +1,6 @@
 
 services:
-  agent:
+  agent-all-in-one:
     labels:
       logs.agent.grafana.com/log-format: logfmt
       app: grafana-agent

docker-compose/common/compose-include/agent-collect-logs.yaml

@@ -1,8 +1,9 @@
 
 services:
-  agent:
+  agent-logs:
     labels:
-      - logs.agent.grafana.com/log-format=logfmt
+      logs.agent.grafana.com/log-format: logfmt
+      app: grafana-agent
     depends_on:
       gateway:
         condition: service_healthy

docker-compose/common/compose-include/agent-collect-metrics.yaml

@@ -1,6 +1,9 @@
 
 services:
-  agent:
+  agent-metrics:
+    labels:
+      logs.agent.grafana.com/log-format: logfmt
+      app: grafana-agent
     depends_on:
       gateway:
         condition: service_healthy

docker-compose/common/compose-include/agent-collect-profiles.yaml

@@ -1,6 +1,9 @@
 
 services:
-  agent:
+  agent-profiles:
+    labels:
+      logs.agent.grafana.com/log-format: logfmt
+      app: grafana-agent
     depends_on:
       gateway:
         condition: service_started

docker-compose/common/compose-include/agent-collect-traces.yaml

@@ -1,6 +1,9 @@
 
 services:
-  agent:
+  agent-traces:
+    labels:
+      logs.agent.grafana.com/log-format: logfmt
+      app: grafana-agent
     depends_on:
       gateway:
         condition: service_healthy

docker-compose/common/compose-include/load-rules-to-mimir.yaml

@@ -0,0 +1,21 @@
+services:
+  load-rules-to-mimir:
+    depends_on:
+      gateway:
+        condition: service_healthy
+    image: ${MIMIRTOOL_IMAGE:-docker.io/grafana/mimirtool:latest}
+    volumes:
+      - ../../../monitoring-mixins/agent-flow-mixin/deploy/agent-flow-mixin-alerts.yaml:/agent-flow-mixin-alerts.yaml
+      - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/loki-mixin-rules.yaml
+      - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/loki-mixin-alerts.yaml
+      - ../../../monitoring-mixins/memcached-mixin/deploy/memcached-mixin-alerts.yaml:/memcached-mixin-alerts.yaml
+      - ../../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-rules.yaml:/mimir-mixin-rules.yaml
+      - ../../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-alerts.yaml:/mimir-mixin-alerts.yaml
+      - ../../../monitoring-mixins/pyroscope-mixin/deploy/pyroscope-mixin-rules.yaml:/pyroscope-mixin-rules.yaml
+      # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/tempo-mixin-rules.yaml
+      # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/tempo-mixin-alerts.yaml
+    environment:
+      - MIMIR_ADDRESS=http://gateway:8080
+      - MIMIR_TENANT_ID=anonymous
+    command: >-
+      rules load agent-flow-mixin-alerts.yaml /loki-mixin-rules.yaml /loki-mixin-alerts.yaml /memcached-mixin-alerts.yaml /mimir-mixin-rules.yaml /mimir-mixin-alerts.yaml /pyroscope-mixin-rules.yaml

docker-compose/common/compose-include/option-to-collect-metrics.yaml

@@ -0,0 +1,31 @@
+
+# Note: 
+# include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later.
+include:
+  - path: agent-collect-metrics.yaml
+  - path: load-rules-to-mimir.yaml
+
+services:
+  mimir:
+    depends_on:
+      minio:
+        condition: service_healthy
+    image: ${MIMIR_IMAGE:-docker.io/grafana/mimir:latest}
+    volumes:
+      - ../config/mimir:/etc/mimir
+    command:
+      - -config.file=/etc/mimir/monolithic-mode-metrics.yaml
+      - -target=all
+      - -config.expand-env=true
+    healthcheck:
+      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/ready || exit 1" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    # expose 38080 port so we can directly access mimir inside container
+    ports:
+      - "38080:8080"
+    networks:
+      default:
+        aliases:
+          - mimir-memberlist

docker-compose/common/config/agent-flow/traces.river

@@ -18,18 +18,6 @@ module.file "lgtmp_provider_local" {
 	}
 }
 
-/********************************************
- * Metrics
- ********************************************/
-
-module.file "metrics_primary" {
-	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river"
-
-	arguments {
-		forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver]
-	}
-}
-
 /********************************************
  * Traces
  ********************************************/

docker-compose/common/config/grafana/provisioning/datasources/.gitignore

@@ -1,2 +1,3 @@
 *
 !.gitignore
+!datasources.yaml

docker-compose/common/config/mimir/runtime.yaml

@@ -15,4 +15,5 @@ overrides:
     # ingestion_tenant_shard_size: 9
     max_global_series_per_user: 1500000
     max_fetched_series_per_query: 100000
-    native_histograms_ingestion_enabled: true
+    native_histograms_ingestion_enabled: true
+    ruler_max_rules_per_rule_group: 50

docker-compose/microservices-mode/logs/docker-compose.yaml

@@ -8,17 +8,21 @@ version: '3.9'
 # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later.
 include:
   - path: ../../common/compose-include/minio.yaml
-  - path: ../../common/compose-include/agent-collect-logs.yaml
   - path: ../../common/compose-include/memcached.yaml
+  - path: ../../common/compose-include/agent-collect-logs.yaml
+  # optional
+  - path: ../../common/compose-include/option-to-collect-metrics.yaml
 
+# https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md
 x-labels: &loki-labels
-  - logs.agent.grafana.com/log-format=json
+  logs.agent.grafana.com/log-format: json
 
 services:
   gateway:
+    # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md
     labels:
-      - logs.agent.grafana.com/scrape=false
-      - logs.agent.grafana.com/log-format=nginx
+      metrics.agent.grafana.com/scrape: false
+      logs.agent.grafana.com/log-format: nginx
     depends_on:
       distributor:
         condition: service_healthy
@@ -27,6 +31,7 @@ services:
       - ../../common/config/nginx/10-default-lgtmp.envsh:/docker-entrypoint.d/10-default-lgtmp.envsh
       - ../../common/config/nginx/nginx.conf:/etc/nginx/templates/nginx.conf.template
       - ../../common/config/loki/gateway_loki.conf:/etc/nginx/templates/gateway_loki.conf.template
+      - ../../common/config/mimir/gateway_mimir.conf:/etc/nginx/templates/gateway_mimir.conf.template
     environment:
       - LOKI_DISTRIBUTOR_HOST=distributor
       - LOKI_INGESTER_HOST=ingester
@@ -160,15 +165,19 @@ services:
 
   grafana:
     labels:
-      - logs.agent.grafana.com/log-format=logfmt
+      logs.agent.grafana.com/log-format: logfmt
     image: ${GRAFANA_IMAGE:-docker.io/grafana/grafana:latest}
     command:
       - --config=/etc/grafana-config/grafana.ini
     volumes:
       - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini
       - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards
       - ../../common/config/grafana/provisioning:/etc/grafana/provisioning
-      - ../../common/config/loki/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml # load datasources
+      - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin
+      - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin
+      - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
+      - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin
+      - ../../../monitoring-mixins/memcached-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/memcached-mixin
     environment:
       - GF_LOG_LEVEL=warn
       - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password}