diff --git a/docker-compose/common/compose-include/agent-collect-all.yaml b/docker-compose/common/compose-include/agent-collect-all.yaml index 1b6c84bb..e9ea8e19 100644 --- a/docker-compose/common/compose-include/agent-collect-all.yaml +++ b/docker-compose/common/compose-include/agent-collect-all.yaml @@ -1,6 +1,6 @@ services: - agent: + agent-all-in-one: labels: logs.agent.grafana.com/log-format: logfmt app: grafana-agent diff --git a/docker-compose/common/compose-include/agent-collect-logs.yaml b/docker-compose/common/compose-include/agent-collect-logs.yaml index 63dc5efb..4f3e5221 100644 --- a/docker-compose/common/compose-include/agent-collect-logs.yaml +++ b/docker-compose/common/compose-include/agent-collect-logs.yaml @@ -1,8 +1,9 @@ services: - agent: + agent-logs: labels: - - logs.agent.grafana.com/log-format=logfmt + logs.agent.grafana.com/log-format: logfmt + app: grafana-agent depends_on: gateway: condition: service_healthy diff --git a/docker-compose/common/compose-include/agent-collect-metrics.yaml b/docker-compose/common/compose-include/agent-collect-metrics.yaml index 59a8cfba..d180ce97 100644 --- a/docker-compose/common/compose-include/agent-collect-metrics.yaml +++ b/docker-compose/common/compose-include/agent-collect-metrics.yaml @@ -1,6 +1,9 @@ services: - agent: + agent-metrics: + labels: + logs.agent.grafana.com/log-format: logfmt + app: grafana-agent depends_on: gateway: condition: service_healthy diff --git a/docker-compose/common/compose-include/agent-collect-profiles.yaml b/docker-compose/common/compose-include/agent-collect-profiles.yaml index 636ab91c..7519b678 100644 --- a/docker-compose/common/compose-include/agent-collect-profiles.yaml +++ b/docker-compose/common/compose-include/agent-collect-profiles.yaml @@ -1,6 +1,9 @@ services: - agent: + agent-profiles: + labels: + logs.agent.grafana.com/log-format: logfmt + app: grafana-agent depends_on: gateway: condition: service_started diff --git a/docker-compose/common/compose-include/agent-collect-traces.yaml b/docker-compose/common/compose-include/agent-collect-traces.yaml index c038c83f..147b5c3a 100644 --- a/docker-compose/common/compose-include/agent-collect-traces.yaml +++ b/docker-compose/common/compose-include/agent-collect-traces.yaml @@ -1,6 +1,9 @@ services: - agent: + agent-traces: + labels: + logs.agent.grafana.com/log-format: logfmt + app: grafana-agent depends_on: gateway: condition: service_healthy diff --git a/docker-compose/common/compose-include/load-mimir-rule.yaml b/docker-compose/common/compose-include/load-mimir-rule.yaml deleted file mode 100644 index 4d0f5751..00000000 --- a/docker-compose/common/compose-include/load-mimir-rule.yaml +++ /dev/null @@ -1,17 +0,0 @@ -services: - load-mimir-rules: - depends_on: - gateway: - condition: service_healthy - image: ${MIMIRTOOL_IMAGE:-docker.io/grafana/mimirtool:latest} - volumes: - - ../../../monitoring-mixins/agent-flow-mixin/deploy/agent-flow-mixin-alerts.yaml:/agent-flow-mixin-alerts.yaml - - ../../../monitoring-mixins/memcached-mixin/deploy/memcached-mixin-alerts.yaml:/memcached-mixin-alerts.yaml - - ../../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-rules.yaml:/mimir-mixin-rules.yaml - - ../../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-alerts.yaml:/mimir-mixin-alerts.yaml - environment: - - MIMIR_ADDRESS=http://gateway:8080 - - MIMIR_TENANT_ID=anonymous - command: >- - rules load /agent-flow-mixin-alerts.yaml /memcached-mixin-alerts.yaml /mimir-mixin-rules.yaml /mimir-mixin-alerts.yaml - \ No newline at end of file diff --git a/docker-compose/common/compose-include/load-rules-to-mimir.yaml b/docker-compose/common/compose-include/load-rules-to-mimir.yaml new file mode 100644 index 00000000..59d4d113 --- /dev/null +++ b/docker-compose/common/compose-include/load-rules-to-mimir.yaml @@ -0,0 +1,21 @@ +services: + load-rules-to-mimir: + depends_on: + gateway: + condition: service_healthy + image: ${MIMIRTOOL_IMAGE:-docker.io/grafana/mimirtool:latest} + volumes: + - ../../../monitoring-mixins/agent-flow-mixin/deploy/agent-flow-mixin-alerts.yaml:/agent-flow-mixin-alerts.yaml + - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/loki-mixin-rules.yaml + - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/loki-mixin-alerts.yaml + - ../../../monitoring-mixins/memcached-mixin/deploy/memcached-mixin-alerts.yaml:/memcached-mixin-alerts.yaml + - ../../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-rules.yaml:/mimir-mixin-rules.yaml + - ../../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-alerts.yaml:/mimir-mixin-alerts.yaml + - ../../../monitoring-mixins/pyroscope-mixin/deploy/pyroscope-mixin-rules.yaml:/pyroscope-mixin-rules.yaml + # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/tempo-mixin-rules.yaml + # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/tempo-mixin-alerts.yaml + environment: + - MIMIR_ADDRESS=http://gateway:8080 + - MIMIR_TENANT_ID=anonymous + command: >- + rules load agent-flow-mixin-alerts.yaml /loki-mixin-rules.yaml /loki-mixin-alerts.yaml /memcached-mixin-alerts.yaml /mimir-mixin-rules.yaml /mimir-mixin-alerts.yaml /pyroscope-mixin-rules.yaml diff --git a/docker-compose/common/compose-include/load-tempo-rule.yaml b/docker-compose/common/compose-include/load-tempo-rule.yaml deleted file mode 100644 index 3f85c961..00000000 --- a/docker-compose/common/compose-include/load-tempo-rule.yaml +++ /dev/null @@ -1,15 +0,0 @@ -services: - load-tempo-rules: - depends_on: - gateway: - condition: service_healthy - image: ${MIMIRTOOL_IMAGE:-docker.io/grafana/mimirtool:latest} - volumes: - - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/tempo-mixin-rules.yaml - - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/tempo-mixin-alerts.yaml - environment: - - MIMIR_ADDRESS=http://gateway:8080 - - MIMIR_TENANT_ID=anonymous - command: >- - rules load /tempo-mixin-rules.yaml /tempo-mixin-alerts.yaml - \ No newline at end of file diff --git a/docker-compose/common/compose-include/option-to-collect-metrics.yaml b/docker-compose/common/compose-include/option-to-collect-metrics.yaml new file mode 100644 index 00000000..61df0c4a --- /dev/null +++ b/docker-compose/common/compose-include/option-to-collect-metrics.yaml @@ -0,0 +1,31 @@ + +# Note: +# include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. +include: + - path: agent-collect-metrics.yaml + - path: load-rules-to-mimir.yaml + +services: + mimir: + depends_on: + minio: + condition: service_healthy + image: ${MIMIR_IMAGE:-docker.io/grafana/mimir:latest} + volumes: + - ../config/mimir:/etc/mimir + command: + - -config.file=/etc/mimir/monolithic-mode-metrics.yaml + - -target=all + - -config.expand-env=true + healthcheck: + test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/ready || exit 1" ] + interval: 10s + timeout: 5s + retries: 5 + # expose 38080 port so we can directly access mimir inside container + ports: + - "38080:8080" + networks: + default: + aliases: + - mimir-memberlist diff --git a/docker-compose/common/config/agent-flow/traces.river b/docker-compose/common/config/agent-flow/traces.river index 86accf35..fdcbdf0f 100644 --- a/docker-compose/common/config/agent-flow/traces.river +++ b/docker-compose/common/config/agent-flow/traces.river @@ -18,18 +18,6 @@ module.file "lgtmp_provider_local" { } } -/******************************************** - * Metrics - ********************************************/ - -module.file "metrics_primary" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river" - - arguments { - forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver] - } -} - /******************************************** * Traces ********************************************/ diff --git a/docker-compose/common/config/grafana/provisioning/datasources/.gitignore b/docker-compose/common/config/grafana/provisioning/datasources/.gitignore index d6b7ef32..1e800a9a 100644 --- a/docker-compose/common/config/grafana/provisioning/datasources/.gitignore +++ b/docker-compose/common/config/grafana/provisioning/datasources/.gitignore @@ -1,2 +1,3 @@ * !.gitignore +!datasources.yaml \ No newline at end of file diff --git a/docker-compose/monolithic-mode/all-in-one/config/grafana/datasources.yaml b/docker-compose/common/config/grafana/provisioning/datasources/datasources.yaml similarity index 100% rename from docker-compose/monolithic-mode/all-in-one/config/grafana/datasources.yaml rename to docker-compose/common/config/grafana/provisioning/datasources/datasources.yaml diff --git a/docker-compose/common/config/loki/datasources.yaml b/docker-compose/common/config/loki/datasources.yaml deleted file mode 100644 index ee9074f6..00000000 --- a/docker-compose/common/config/loki/datasources.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: 1 - -deleteDatasources: -- name: Metrics - uid: metrics -- name: Logs - uid: logs - -datasources: -# Mimir for metrics -- name: Metrics - type: prometheus - uid: metrics - access: proxy - orgId: 1 - url: http://gateway:8080/prometheus - basicAuth: false - isDefault: false - version: 1 - editable: true - -# Loki for logs -- name: Logs - type: loki - access: proxy - orgId: 1 - uid: logs - url: http://gateway:3100 - basicAuth: false - isDefault: true - version: 1 - editable: true diff --git a/docker-compose/common/config/mimir/datasources.yaml b/docker-compose/common/config/mimir/datasources.yaml deleted file mode 100644 index a3527f25..00000000 --- a/docker-compose/common/config/mimir/datasources.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: 1 - -deleteDatasources: -- name: Metrics - uid: metrics -- name: Logs - uid: logs - -datasources: -# Mimir for metrics -- name: Metrics - type: prometheus - uid: metrics - access: proxy - orgId: 1 - url: http://gateway:8080/prometheus - basicAuth: false - isDefault: true # Note: default datasources - version: 1 - editable: true - -# Loki for logs -- name: Logs - type: loki - access: proxy - orgId: 1 - uid: logs - url: http://gateway:3100 - basicAuth: false - isDefault: false - version: 1 - editable: true diff --git a/docker-compose/common/config/mimir/runtime.yaml b/docker-compose/common/config/mimir/runtime.yaml index 3395a101..a5991cc7 100644 --- a/docker-compose/common/config/mimir/runtime.yaml +++ b/docker-compose/common/config/mimir/runtime.yaml @@ -15,4 +15,5 @@ overrides: # ingestion_tenant_shard_size: 9 max_global_series_per_user: 1500000 max_fetched_series_per_query: 100000 - native_histograms_ingestion_enabled: true \ No newline at end of file + native_histograms_ingestion_enabled: true + ruler_max_rules_per_rule_group: 50 \ No newline at end of file diff --git a/docker-compose/common/config/pyroscope/datasources.yaml b/docker-compose/common/config/pyroscope/datasources.yaml deleted file mode 100644 index e61e4dde..00000000 --- a/docker-compose/common/config/pyroscope/datasources.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: 1 - -deleteDatasources: -- name: Profiles - uid: profiles - -datasources: -# Pyroscope for profiles -- name: Profiles - type: grafana-pyroscope-datasource - access: proxy - orgId: 1 - uid: profiles - url: http://gateway:4040 - basicAuth: false - isDefault: true - version: 1 - editable: true diff --git a/docker-compose/common/config/tempo/datasources.yaml b/docker-compose/common/config/tempo/datasources.yaml deleted file mode 100644 index d9a14053..00000000 --- a/docker-compose/common/config/tempo/datasources.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: 1 - -deleteDatasources: -- name: Metrics - uid: metrics -- name: Logs - uid: logs -- name: Traces - uid: traces - -datasources: -# Mimir for metrics -- name: Metrics - type: prometheus - uid: metrics - access: proxy - orgId: 1 - url: http://gateway:8080/prometheus - basicAuth: false - isDefault: false - version: 1 - editable: true - jsonData: - prometheusType: Mimir - exemplarTraceIdDestinations: - - name: traceID - datasourceUid: traces - -# Loki for logs -- name: Logs - type: loki - access: proxy - orgId: 1 - uid: logs - url: http://gateway:3100 - basicAuth: false - isDefault: false - version: 1 - editable: true - -# Tempo for traces -- name: Traces - type: tempo - access: proxy - orgId: 1 - uid: traces - url: http://gateway:3200 - basicAuth: false - isDefault: true # Note: default datasources - version: 1 - editable: true - apiVersion: 1 - jsonData: - search: - hide: false - nodeGraph: - enabled: true - serviceMap: - datasourceUid: metrics - traceQuery: - timeShiftEnabled: true - spanStartTimeShift: '-30m' - spanEndTimeShift: '30m' - spanBar: - type: 'Tag' - tag: 'http.path' - tracesToMetrics: - datasourceUid: metrics - spanStartTimeShift: '-30m' - spanEndTimeShift: '30m' - tags: [{ key: 'service.name', value: 'service' }] - queries: - - name: '(R) Rate' - query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[$$__rate_interval]))' - - name: '(E) Error Rate' - query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags, status_code="STATUS_CODE_ERROR"}[$$__rate_interval]))' - - name: '(D) Duration' - query: 'histogram_quantile(0.9, sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[$$__rate_interval])) by (le))' diff --git a/docker-compose/microservices-mode/logs/docker-compose.yaml b/docker-compose/microservices-mode/logs/docker-compose.yaml index a1bbf366..e4a592ea 100644 --- a/docker-compose/microservices-mode/logs/docker-compose.yaml +++ b/docker-compose/microservices-mode/logs/docker-compose.yaml @@ -8,17 +8,21 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-logs.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-logs.yaml + # optional + - path: ../../common/compose-include/option-to-collect-metrics.yaml +# https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md x-labels: &loki-labels - - logs.agent.grafana.com/log-format=json + logs.agent.grafana.com/log-format: json services: gateway: + # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md labels: - - logs.agent.grafana.com/scrape=false - - logs.agent.grafana.com/log-format=nginx + metrics.agent.grafana.com/scrape: false + logs.agent.grafana.com/log-format: nginx depends_on: distributor: condition: service_healthy @@ -27,6 +31,7 @@ services: - ../../common/config/nginx/10-default-lgtmp.envsh:/docker-entrypoint.d/10-default-lgtmp.envsh - ../../common/config/nginx/nginx.conf:/etc/nginx/templates/nginx.conf.template - ../../common/config/loki/gateway_loki.conf:/etc/nginx/templates/gateway_loki.conf.template + - ../../common/config/mimir/gateway_mimir.conf:/etc/nginx/templates/gateway_mimir.conf.template environment: - LOKI_DISTRIBUTOR_HOST=distributor - LOKI_INGESTER_HOST=ingester @@ -160,7 +165,7 @@ services: grafana: labels: - - logs.agent.grafana.com/log-format=logfmt + logs.agent.grafana.com/log-format: logfmt image: ${GRAFANA_IMAGE:-docker.io/grafana/grafana:latest} command: - --config=/etc/grafana-config/grafana.ini @@ -168,7 +173,11 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/loki/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml # load datasources + - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin + - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin + - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin + - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin + - ../../../monitoring-mixins/memcached-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/memcached-mixin environment: - GF_LOG_LEVEL=warn - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password} diff --git a/docker-compose/microservices-mode/metrics/docker-compose.yaml b/docker-compose/microservices-mode/metrics/docker-compose.yaml index 7504d0b2..0d48f5dd 100644 --- a/docker-compose/microservices-mode/metrics/docker-compose.yaml +++ b/docker-compose/microservices-mode/metrics/docker-compose.yaml @@ -8,9 +8,9 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-metrics.yaml - - path: ../../common/compose-include/load-mimir-rule.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-metrics.yaml + - path: ../../common/compose-include/load-rules-to-mimir.yaml services: gateway: @@ -214,7 +214,6 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/mimir/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin diff --git a/docker-compose/microservices-mode/profiles/docker-compose.yaml b/docker-compose/microservices-mode/profiles/docker-compose.yaml index 4fcc4cb8..6ca81d78 100644 --- a/docker-compose/microservices-mode/profiles/docker-compose.yaml +++ b/docker-compose/microservices-mode/profiles/docker-compose.yaml @@ -8,7 +8,10 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml + - path: ../../common/compose-include/memcached.yaml - path: ../../common/compose-include/agent-collect-profiles.yaml + # optional + - path: ../../common/compose-include/option-to-collect-metrics.yaml # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md x-labels: &profiles-labels @@ -18,6 +21,9 @@ x-labels: &profiles-labels services: gateway: + # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md + labels: + metrics.agent.grafana.com/scrape: false depends_on: distributor: condition: service_healthy @@ -27,6 +33,7 @@ services: - ../../common/config/nginx/10-default-lgtmp.envsh:/docker-entrypoint.d/10-default-lgtmp.envsh - ../../common/config/nginx/nginx.conf:/etc/nginx/templates/nginx.conf.template - ../../common/config/pyroscope/gateway_pyroscope.conf:/etc/nginx/templates/gateway_pyroscope.conf.template + - ../../common/config/mimir/gateway_mimir.conf:/etc/nginx/templates/gateway_mimir.conf.template environment: - PYROSCOPE_DISTRIBUTOR_HOST=distributor - PYROSCOPE_QUERY_FRONTEND_HOST=query-frontend @@ -211,7 +218,11 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/pyroscope/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin + - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin + - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin + - ../../../monitoring-mixins/pyroscope-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/pyroscope-mixin + - ../../../monitoring-mixins/memcached-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/memcached-mixin environment: - GF_LOG_LEVEL=warn - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password} diff --git a/docker-compose/microservices-mode/traces/docker-compose.yaml b/docker-compose/microservices-mode/traces/docker-compose.yaml index 7d1e301e..c7ea5491 100644 --- a/docker-compose/microservices-mode/traces/docker-compose.yaml +++ b/docker-compose/microservices-mode/traces/docker-compose.yaml @@ -8,13 +8,13 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-traces.yaml - - path: ../../common/compose-include/load-mimir-rule.yaml - - path: ../../common/compose-include/load-tempo-rule.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-traces.yaml + # optional + - path: ../../common/compose-include/option-to-collect-metrics.yaml x-environment: &jaeger-environment - JAEGER_AGENT_HOST: agent + JAEGER_AGENT_HOST: agent-traces JAEGER_AGENT_PORT: 6831 JAEGER_SAMPLER_TYPE: const JAEGER_SAMPLER_PARAM: 1 @@ -135,34 +135,7 @@ services: <<: *jaeger-environment JAEGER_TAGS: app=compactor - mimir: - depends_on: - minio: - condition: service_started - image: ${MIMIR_IMAGE:-docker.io/grafana/mimir:latest} - volumes: - - ../../common/config/mimir:/etc/mimir - command: - - -config.file=/etc/mimir/monolithic-mode-metrics.yaml - - -target=all - - -config.expand-env=true - healthcheck: - test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/ready || exit 1" ] - interval: 10s - timeout: 5s - retries: 5 - # expose 38080 port so we can directly access mimir inside container - ports: - - "38080:8080" - networks: - default: - aliases: - - mimir-memberlist - grafana: - depends_on: - agent: - condition: service_started image: ${GRAFANA_IMAGE:-docker.io/grafana/grafana:latest} command: - --config=/etc/grafana-config/grafana.ini @@ -170,7 +143,6 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/tempo/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml # load datasources - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin # - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin @@ -179,6 +151,6 @@ services: environment: - GF_LOG_LEVEL=warn - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password} - - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor traceqlSearch correlations metricsSummary traceToMetrics ports: - "3000:3000" diff --git a/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml b/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml index 08961644..11a916d6 100644 --- a/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml +++ b/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml @@ -8,8 +8,9 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-all.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-all.yaml + - path: ../../common/compose-include/load-rules-to-mimir.yaml # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md x-labels: &profiles-labels @@ -51,59 +52,6 @@ services: timeout: 5s retries: 10 - mimir: - labels: - <<: *profiles-labels - profiles.agent.grafana.com/service_name: mimir - logs.agent.grafana.com/log-format: logfmt - depends_on: - minio: - condition: service_healthy - image: ${MIMIR_IMAGE:-docker.io/grafana/mimir:latest} - volumes: - - ../../common/config/mimir:/etc/mimir - command: - - -config.file=/etc/mimir/monolithic-mode-metrics.yaml - - -target=all - - -config.expand-env=true - - -ruler.max-rules-per-rule-group=50 - environment: - <<: *jaeger-environment - JAEGER_TAGS: app=mimir - healthcheck: - test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/ready || exit 1" ] - interval: 10s - timeout: 5s - retries: 5 - # expose 38080 port so we can directly access mimir inside container - ports: - - "38080:8080" - networks: - default: - aliases: - - mimir-memberlist - - load-rules-to-mimir: - depends_on: - mimir: - condition: service_healthy - image: ${MIMIRTOOL_IMAGE:-docker.io/grafana/mimirtool:latest} - volumes: - - ../../../monitoring-mixins/agent-flow-mixin/deploy/agent-flow-mixin-alerts.yaml:/agent-flow-mixin-alerts.yaml - - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/loki-mixin-rules.yaml - - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/loki-mixin-alerts.yaml - - ../../../monitoring-mixins/memcached-mixin/deploy/memcached-mixin-alerts.yaml:/memcached-mixin-alerts.yaml - - ../../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-rules.yaml:/mimir-mixin-rules.yaml - - ../../../monitoring-mixins/mimir-mixin/deploy/mimir-mixin-alerts.yaml:/mimir-mixin-alerts.yaml - - ../../../monitoring-mixins/pyroscope-mixin/deploy/pyroscope-mixin-rules.yaml:/pyroscope-mixin-rules.yaml - # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/tempo-mixin-rules.yaml - # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/tempo-mixin-alerts.yaml - environment: - - MIMIR_ADDRESS=http://mimir:8080 - - MIMIR_TENANT_ID=anonymous - command: >- - rules load agent-flow-mixin-alerts.yaml /loki-mixin-rules.yaml /loki-mixin-alerts.yaml /memcached-mixin-alerts.yaml /mimir-mixin-rules.yaml /mimir-mixin-alerts.yaml /pyroscope-mixin-rules.yaml - loki: # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md labels: @@ -166,6 +114,38 @@ services: ports: - "33200:3200" + mimir: + labels: + <<: *profiles-labels + profiles.agent.grafana.com/service_name: mimir + logs.agent.grafana.com/log-format: logfmt + depends_on: + minio: + condition: service_healthy + image: ${MIMIR_IMAGE:-docker.io/grafana/mimir:latest} + volumes: + - ../../common/config/mimir:/etc/mimir + command: + - -config.file=/etc/mimir/monolithic-mode-metrics.yaml + - -target=all + - -config.expand-env=true + - -ruler.max-rules-per-rule-group=50 + environment: + <<: *jaeger-environment + JAEGER_TAGS: app=mimir + healthcheck: + test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/ready || exit 1" ] + interval: 10s + timeout: 5s + retries: 5 + # expose 38080 port so we can directly access mimir inside container + ports: + - "38080:8080" + networks: + default: + aliases: + - mimir-memberlist + pyroscope: labels: - logs.agent.grafana.com/log-format=logfmt @@ -204,7 +184,6 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ./config/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin diff --git a/docker-compose/monolithic-mode/logs/docker-compose.yaml b/docker-compose/monolithic-mode/logs/docker-compose.yaml index 12e288ae..6c94f99f 100644 --- a/docker-compose/monolithic-mode/logs/docker-compose.yaml +++ b/docker-compose/monolithic-mode/logs/docker-compose.yaml @@ -8,14 +8,17 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-logs.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-logs.yaml + # optional + - path: ../../common/compose-include/option-to-collect-metrics.yaml services: gateway: + # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md labels: - # - logs.agent.grafana.com/scrape=false - - logs.agent.grafana.com/log-format=nginx + metrics.agent.grafana.com/scrape: false + logs.agent.grafana.com/log-format: nginx depends_on: loki: condition: service_healthy @@ -24,6 +27,7 @@ services: - ../../common/config/nginx/10-default-lgtmp.envsh:/docker-entrypoint.d/10-default-lgtmp.envsh - ../../common/config/nginx/nginx.conf:/etc/nginx/templates/nginx.conf.template - ../../common/config/loki/gateway_loki.conf:/etc/nginx/templates/gateway_loki.conf.template + - ../../common/config/mimir/gateway_mimir.conf:/etc/nginx/templates/gateway_mimir.conf.template healthcheck: test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ || exit 1" ] interval: 10s @@ -66,7 +70,11 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/loki/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin + - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin + - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin + - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin + - ../../../monitoring-mixins/memcached-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/memcached-mixin environment: - GF_LOG_LEVEL=warn - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password} diff --git a/docker-compose/monolithic-mode/metrics/docker-compose.yaml b/docker-compose/monolithic-mode/metrics/docker-compose.yaml index 8b3576ce..3d73832f 100644 --- a/docker-compose/monolithic-mode/metrics/docker-compose.yaml +++ b/docker-compose/monolithic-mode/metrics/docker-compose.yaml @@ -8,9 +8,9 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-metrics.yaml - - path: ../../common/compose-include/load-mimir-rule.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-metrics.yaml + - path: ../../common/compose-include/load-rules-to-mimir.yaml services: gateway: @@ -63,7 +63,6 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/mimir/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin diff --git a/docker-compose/monolithic-mode/profiles/docker-compose.yaml b/docker-compose/monolithic-mode/profiles/docker-compose.yaml index 8f47e1ec..f914d893 100644 --- a/docker-compose/monolithic-mode/profiles/docker-compose.yaml +++ b/docker-compose/monolithic-mode/profiles/docker-compose.yaml @@ -8,7 +8,10 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml + - path: ../../common/compose-include/memcached.yaml - path: ../../common/compose-include/agent-collect-profiles.yaml + # optional + - path: ../../common/compose-include/option-to-collect-metrics.yaml # # for git remote # # https://github.com/docker/compose/pull/10811 # # https://docs.docker.com/engine/reference/commandline/build/#git-repositories @@ -27,6 +30,9 @@ x-labels: &profiles-labels services: gateway: + # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md + labels: + metrics.agent.grafana.com/scrape: false depends_on: pyroscope: condition: service_healthy @@ -35,6 +41,7 @@ services: - ../../common/config/nginx/10-default-lgtmp.envsh:/docker-entrypoint.d/10-default-lgtmp.envsh - ../../common/config/nginx/nginx.conf:/etc/nginx/templates/nginx.conf.template - ../../common/config/pyroscope/gateway_pyroscope.conf:/etc/nginx/templates/gateway_pyroscope.conf.template + - ../../common/config/mimir/gateway_mimir.conf:/etc/nginx/templates/gateway_mimir.conf.template healthcheck: test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:4040/ || exit 1" ] interval: 10s @@ -78,7 +85,11 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/pyroscope/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin + - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin + - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin + - ../../../monitoring-mixins/pyroscope-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/pyroscope-mixin + - ../../../monitoring-mixins/memcached-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/memcached-mixin environment: - GF_LOG_LEVEL=warn - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password} diff --git a/docker-compose/monolithic-mode/traces/docker-compose.yaml b/docker-compose/monolithic-mode/traces/docker-compose.yaml index d71b8cd6..f81c9ec0 100644 --- a/docker-compose/monolithic-mode/traces/docker-compose.yaml +++ b/docker-compose/monolithic-mode/traces/docker-compose.yaml @@ -8,13 +8,13 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-traces.yaml - - path: ../../common/compose-include/load-mimir-rule.yaml - - path: ../../common/compose-include/load-tempo-rule.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-traces.yaml + # optional + - path: ../../common/compose-include/option-to-collect-metrics.yaml x-environment: &jaeger-environment - JAEGER_AGENT_HOST: agent + JAEGER_AGENT_HOST: agent-traces JAEGER_AGENT_PORT: 6831 JAEGER_SAMPLER_TYPE: const JAEGER_SAMPLER_PARAM: 1 @@ -64,30 +64,6 @@ services: ports: - "33200:3200" - mimir: - depends_on: - minio: - condition: service_started - image: ${MIMIR_IMAGE:-docker.io/grafana/mimir:latest} - volumes: - - ../../common/config/mimir:/etc/mimir - command: - - -config.file=/etc/mimir/monolithic-mode-metrics.yaml - - -target=all - - -config.expand-env=true - healthcheck: - test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/ready || exit 1" ] - interval: 10s - timeout: 5s - retries: 5 - # expose 38080 port so we can directly access mimir inside container - ports: - - "38080:8080" - networks: - default: - aliases: - - mimir-memberlist - grafana: depends_on: minio: @@ -99,7 +75,6 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/tempo/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin # - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin @@ -108,6 +83,6 @@ services: environment: - GF_LOG_LEVEL=warn - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password} - - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor traceqlSearch correlations metricsSummary traceToMetrics ports: - "3000:3000" diff --git a/docker-compose/read-write-mode/logs/docker-compose.yaml b/docker-compose/read-write-mode/logs/docker-compose.yaml index 3704583e..31c4acf1 100644 --- a/docker-compose/read-write-mode/logs/docker-compose.yaml +++ b/docker-compose/read-write-mode/logs/docker-compose.yaml @@ -8,17 +8,20 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-logs.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-logs.yaml + # optional + - path: ../../common/compose-include/option-to-collect-metrics.yaml x-labels: &loki-labels - - logs.agent.grafana.com/log-format=json + logs.agent.grafana.com/log-format: json services: gateway: + # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md labels: - - logs.agent.grafana.com/scrape=false - - logs.agent.grafana.com/log-format=nginx + metrics.agent.grafana.com/scrape: false + logs.agent.grafana.com/log-format: nginx depends_on: loki-write: condition: service_healthy @@ -27,6 +30,7 @@ services: - ../../common/config/nginx/10-default-lgtmp.envsh:/docker-entrypoint.d/10-default-lgtmp.envsh - ../../common/config/nginx/nginx.conf:/etc/nginx/templates/nginx.conf.template - ../../common/config/loki/gateway_loki.conf:/etc/nginx/templates/gateway_loki.conf.template + - ../../common/config/mimir/gateway_mimir.conf:/etc/nginx/templates/gateway_mimir.conf.template environment: - LOKI_DISTRIBUTOR_HOST=loki-write - LOKI_INGESTER_HOST=loki-write @@ -114,7 +118,7 @@ services: grafana: labels: - - logs.agent.grafana.com/log-format=logfmt + logs.agent.grafana.com/log-format: logfmt image: ${GRAFANA_IMAGE:-docker.io/grafana/grafana:latest} command: - --config=/etc/grafana-config/grafana.ini @@ -122,7 +126,11 @@ services: - ../../common/config/grafana/grafana.ini:/etc/grafana-config/grafana.ini - ../../common/config/grafana/dashboards:/var/lib/grafana/dashboards - ../../common/config/grafana/provisioning:/etc/grafana/provisioning - - ../../common/config/loki/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml # load datasources + - ../../../monitoring-mixins/agent-flow-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/agent-flow-mixin + - ../../../monitoring-mixins/go-runtime-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/go-runtime-mixin + - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin + - ../../../monitoring-mixins/mimir-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/mimir-mixin + - ../../../monitoring-mixins/memcached-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/memcached-mixin environment: - GF_LOG_LEVEL=warn - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin_password} diff --git a/docker-compose/read-write-mode/metrics/docker-compose.yaml b/docker-compose/read-write-mode/metrics/docker-compose.yaml index 33aa8649..fd5f9cba 100644 --- a/docker-compose/read-write-mode/metrics/docker-compose.yaml +++ b/docker-compose/read-write-mode/metrics/docker-compose.yaml @@ -8,9 +8,9 @@ version: '3.9' # include is available in Docker Compose version 2.20 and later, and Docker Desktop version 4.22 and later. include: - path: ../../common/compose-include/minio.yaml - - path: ../../common/compose-include/agent-collect-metrics.yaml - - path: ../../common/compose-include/load-mimir-rule.yaml - path: ../../common/compose-include/memcached.yaml + - path: ../../common/compose-include/agent-collect-metrics.yaml + - path: ../../common/compose-include/load-rules-to-mimir.yaml services: gateway: