diff --git a/docker-compose/common/config/agent-flow/modules/docker/README.md b/docker-compose/common/config/agent-flow/modules/docker/README.md index a1d0e6fe..790bbc29 100644 --- a/docker-compose/common/config/agent-flow/modules/docker/README.md +++ b/docker-compose/common/config/agent-flow/modules/docker/README.md @@ -39,3 +39,22 @@ The following service labels are supported for gathering of metrics for docker c | `metrics.agent.grafana.com/job`
or
`prometheus.io/job` | The job label value to use when collecting their metrics. However, it is common to use an integration or community project where rules / dashboards are provided for you. Oftentimes, this provided assets use hard-coded values for a job label i.e. `...{job="integrations/kubernetes/cadvisor"...}` or `...{job="minio-job"...}` setting this annotation to that value will allow the provided asset to work out of the box. | | `metrics.agent.grafana.com/interval`
or
`prometheus.io/interval` | The default interval to scrape is `15s`, this can be override. | | `metrics.agent.grafana.com/timeout`
or
`prometheus.io/timeout` | The default timeout for scraping is `10s`, this can be override. | + +--- + +## Profiles + +The following service labels are supported for docker compose services: + +> The full list of profile types supported by labels is `cpu`, `memory`, `goroutine`, `block`, `mutex` and `fgprof`: + +| Label | Description | +| :--------------- | :-----------| +| `profiles.agent.grafana.com/service_name`
or
`profiles.grafana.com/service_name`
or
`pyroscope.io/service_name` | The special label `service_name` is required and must always be present. If it is not specified, will attempt to infer it from either of the following sources. in this order: | +| `profiles.agent.grafana.com/.scrape`
or
`profiles.grafana.com/.scrape` | Boolean whether or not to scrape. (default is `false`).| +| `profiles.agent.grafana.com/.path`
or
`profiles.grafana.com/.path` | The path to the profile type on the target. | +| `profiles.agent.grafana.com/tenant`
or
`profiles.grafana.com/tenant` | The `tenant` to write profile to. default: (.*) | +| `profiles.agent.grafana.com/scheme`
or
`profiles.grafana.com/scheme` | The default scraping scheme is `http`. | +| `profiles.agent.grafana.com/port`
or
`profiles.grafana.com/port` | the default `port` to scrape is the target port, this can be specified as a single value which would override the scrape port being used for all ports attached to the target, note that even if an target had multiple targets, the relabel_config targets are deduped before scraping | +| `profiles.agent.grafana.com/interval`
or
`profiles.grafana.com/interval`| The default `interval` to scrape is `30s`, this can be override. | +| `profiles.agent.grafana.com/timeout`
or
`profiles.grafana.com/timeout` | The default `timeout` for scraping is `15s`, this can be override. | diff --git a/docker-compose/common/config/agent-flow/modules/docker/profiles/all.river b/docker-compose/common/config/agent-flow/modules/docker/profiles/all.river new file mode 100644 index 00000000..ee44cb6a --- /dev/null +++ b/docker-compose/common/config/agent-flow/modules/docker/profiles/all.river @@ -0,0 +1,27 @@ +/* +Module: profiles-all +Description: Wrapper module to include all Docker containers metric modules +*/ +argument "forward_to" { + comment = "Must be a list(profilessReceiver) where collected profiles should be forwarded to" +} + +argument "tenant" { + comment = "The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex." + optional = true +} + +argument "clustering" { + // Docs: https://grafana.com/docs/agent/latest/flow/concepts/clustering/ + comment = "Whether or not clustering should be enabled (default: true)" +} + +module.file "mf_profiles_auto_scrape" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/profiles/profiles-auto-scrape.river" + + arguments { + forward_to = argument.forward_to.value + tenant = coalesce(argument.tenant.value, ".*") + clustering = coalesce(argument.clustering.value, "true") + } +} diff --git a/docker-compose/common/config/agent-flow/modules/docker/profiles/profiles-auto-scrape.river b/docker-compose/common/config/agent-flow/modules/docker/profiles/profiles-auto-scrape.river new file mode 100644 index 00000000..51afb148 --- /dev/null +++ b/docker-compose/common/config/agent-flow/modules/docker/profiles/profiles-auto-scrape.river @@ -0,0 +1,652 @@ +/* +Module(profiles): Docker Containers Auto-Scraping +Description: Scrapes targets for profiles based on Docker Containers labels + +Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. + + +The full list of profile types supported by labels is "cpu", "memory", "goroutine", "block", "mutex" and "fgprof" + +For example the following labels: + + ``` + profiles.agent.grafana.com/.scrape: true + profiles.agent.grafana.com/.path: /path/to + profiles.agent.grafana.com/service_name: pyroscope + profiles.agent.grafana.com/tenant: primary + profiles.agent.grafana.com/scheme: http + profiles.agent.grafana.com/port: 8080 + profiles.agent.grafana.com/interval: 30s + profiles.agent.grafana.com/timeout: 15s +or + profiles.agent.grafana.com/.scrape: true + profiles.agent.grafana.com/.path: /path/to + profiles.agent.grafana.com/service_name: pyroscope + profiles.agent.grafana.com/tenant: primary + profiles.agent.grafana.com/scheme: http + profiles.agent.grafana.com/port: 8080 + profiles.agent.grafana.com/interval: 30s + profiles.agent.grafana.com/timeout: 15s + ``` +*/ +argument "forward_to" { + comment = "Must be a list(ProfilessReceiver) where collected profile should be forwarded to" +} + +argument "cluster" { + optional = true +} + +argument "namespace" { + optional = true +} + +argument "tenant" { + comment = "The tenant to write profile to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant label, and this can be a regex. (default: (.*))" + optional = true +} + +argument "scrape_interval" { + comment = "How often to scrape profile from the targets (default: 30s)" + optional = true +} + +argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 15s)" + optional = true +} + +argument "clustering" { + // Docs: https://grafana.com/docs/agent/latest/flow/concepts/clustering/ + comment = "Whether or not clustering should be enabled (default: false)" + optional = true +} + +// get the available containers. +discovery.docker "dd_profiles" { + host = "unix:///var/run/docker.sock" + refresh_interval = "15s" + + filter { + name = "status" + values = ["running"] + } +} + +discovery.relabel "dr_docker_profiles" { + targets = discovery.docker.dd_profiles.targets + + // allow resources to declare the protocol to use when collecting profiles + // default value is "http" + rule { + action = "replace" + replacement = "http" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_scheme", + "__meta_docker_container_label_profiles_grafana_com_scheme", + ] + separator = ";" + regex = "^(?:;*)?(https?).*$" + replacement = "$1" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_port", + "__meta_docker_container_label_profiles_grafana_com_port", + ] + separator = ";" + regex = "^(?:;*)?(\\d+).*$" + replacement = "$1" + target_label = "__tmp_port" + } + + rule { + action = "replace" + source_labels = [ + "__address__", + "__tmp_port", + ] + separator = ";" + regex = "^([^:]+)(?::\\d+)?;(\\d+)$" + replacement = "$1:$2" + target_label = "__address__" + } + + // allow resources to declare their profiles should be sent to + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_tenant", + "__meta_docker_container_label_profiles_grafana_com_tenant", + ] + regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" + } + + // allow resources to declare how often their profiles should be collected. + // default value is 30s, the following duration formats are supported (s|m|ms|h|d) + rule { + action = "replace" + replacement = coalesce(argument.scrape_interval.value, "30s") + target_label = "__scrape_interval__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_interval", + "__meta_docker_container_label_profiles_grafana_com_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_interval__" + } + + // allow resources to declare the timeout of the scrape request + // default value is 15s, the following duration formats are supported (s|m|ms|h|d) + rule { + action = "replace" + replacement = coalesce(argument.scrape_timeout.value, "15s") + target_label = "__scrape_timeout__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_interval", + "__meta_docker_container_label_profiles_grafana_com_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_timeout__" + } + + /******************************************** + * Handle Setting Common Labels + ********************************************/ + + // set the cluster label + rule { + action = "replace" + replacement = coalesce(argument.cluster.value, "docker-compose") + target_label = "cluster" + } + + // set the namespace label + rule { + action = "replace" + replacement = coalesce(argument.namespace.value, "monitoring-system") + target_label = "namespace" + } + + // set a default job label to be the namespace/docker_compose_service + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = coalesce(argument.namespace.value, "monitoring-system") + "/$1" + target_label = "job" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "pod" + } + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "service_name" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_app", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_service_name", + "__meta_docker_container_label_profiles_grafana_com_service_name", + "__meta_docker_container_label_pyroscope_io_service_name", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "service_name" + } +} + +/******************************************** + * Pyroscope Scrape CPU + ********************************************/ +discovery.relabel "dr_keep_cpu_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_cpu_scrape", + "__meta_docker_container_label_profiles_grafana_com_cpu_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_cpu_path", + "__meta_docker_container_label_profiles_grafana_com_cpu_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } +} + +pyroscope.scrape "ps_profile_cpu" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-cpu" + targets = discovery.relabel.dr_keep_cpu_targets.output + + clustering { + enabled = coalesce(argument.clustering.value, true) + } + + profiling_config { + profile.process_cpu { + enabled = true + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = false + } + } +} + +/******************************************** + * Pyroscope Scrape Memory + ********************************************/ +discovery.relabel "dr_keep_memory_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_memory_scrape", + "__meta_docker_container_label_profiles_grafana_com_memory_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_memory_path", + "__meta_docker_container_label_profiles_grafana_com_memory_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } +} + +pyroscope.scrape "ps_profile_memory" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-memory" + targets = discovery.relabel.dr_keep_memory_targets.output + + clustering { + enabled = coalesce(argument.clustering.value, true) + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = true + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = false + } + } +} + +/******************************************** + * Pyroscope Scrape Goroutine + ********************************************/ +discovery.relabel "dr_keep_goroutine_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_goroutine_scrape", + "__meta_docker_container_label_profiles_grafana_com_goroutine_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_goroutine_path", + "__meta_docker_container_label_profiles_grafana_com_goroutine_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } +} + +pyroscope.scrape "ps_profile_goroutine" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-goroutine" + targets = discovery.relabel.dr_keep_goroutine_targets.output + + clustering { + enabled = coalesce(argument.clustering.value, true) + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = true + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = false + } + } +} + +/******************************************** + * Pyroscope Scrape Block + ********************************************/ +discovery.relabel "dr_keep_block_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_block_scrape", + "__meta_docker_container_label_profiles_grafana_com_block_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_block_path", + "__meta_docker_container_label_profiles_grafana_com_block_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } +} + +pyroscope.scrape "ps_profile_block" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-block" + targets = discovery.relabel.dr_keep_block_targets.output + + clustering { + enabled = coalesce(argument.clustering.value, true) + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = true + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = false + } + } +} + +/******************************************** + * Pyroscope Scrape Mutex + ********************************************/ +discovery.relabel "dr_keep_mutex_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_mutex_scrape", + "__meta_docker_container_label_profiles_grafana_com_mutex_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_mutex_path", + "__meta_docker_container_label_profiles_grafana_com_mutex_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } +} + +pyroscope.scrape "ps_profile_mutex" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-mutex" + targets = discovery.relabel.dr_keep_mutex_targets.output + + clustering { + enabled = coalesce(argument.clustering.value, true) + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = true + } + + profile.fgprof { + enabled = false + } + } +} + +/******************************************** + * Pyroscope Scrape Fgprof + ********************************************/ +discovery.relabel "dr_keep_fgprof_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_fgprof_scrape", + "__meta_docker_container_label_profiles_grafana_com_fgprof_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_fgprof_path", + "__meta_docker_container_label_profiles_grafana_com_fgprof_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } +} + +pyroscope.scrape "ps_profile_fgprof" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-fgprof" + targets = discovery.relabel.dr_keep_fgprof_targets.output + + clustering { + enabled = coalesce(argument.clustering.value, true) + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = true + } + } +} diff --git a/docker-compose/common/config/agent-flow/monolithic-mode-all.river b/docker-compose/common/config/agent-flow/monolithic-mode-all.river index c1c319f7..7ebb3626 100644 --- a/docker-compose/common/config/agent-flow/monolithic-mode-all.river +++ b/docker-compose/common/config/agent-flow/monolithic-mode-all.river @@ -245,40 +245,11 @@ loki.process "autologging" { * Profiles ********************************************/ -pyroscope.scrape "containers" { - targets = [ - // {"__address__" = "agent:12345", "service_name" = "agent"}, - {"__address__" = "loki:3100", "service_name" = "loki"}, - // {"__address__" = "grafana:6060", "service_name" = "grafana"}, - // {"__address__" = "mimir:8080", "service_name" = "mimir"}, - {"__address__" = "tempo:3200", "service_name" = "tempo"}, - ] - - clustering { - enabled = true - } - - profiling_config { - profile.block { - enabled = true - } - - profile.mutex { - enabled = true - } - - profile.memory { - enabled = true - } +module.file "profiles_primary" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/profiles/all.river" - profile.goroutine { - enabled = true - } - - profile.process_cpu { - enabled = true - } + arguments { + forward_to = [module.file.docker_compose.exports.profiles_receiver] + clustering = true } - - forward_to = [module.file.docker_compose.exports.profiles_receiver] } diff --git a/docker-compose/common/config/agent-flow/profiles.river b/docker-compose/common/config/agent-flow/profiles.river index c0d87c5c..874bda23 100644 --- a/docker-compose/common/config/agent-flow/profiles.river +++ b/docker-compose/common/config/agent-flow/profiles.river @@ -21,36 +21,11 @@ module.file "docker_compose" { * Profiles ********************************************/ -pyroscope.scrape "containers" { - targets = [ - {"__address__" = "grafana:6060", "service_name" = "grafana"}, - ] +module.file "profiles_primary" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/profiles/all.river" - clustering { - enabled = true - } - - profiling_config { - profile.block { - enabled = true - } - - profile.mutex { - enabled = true - } - - profile.memory { - enabled = true - } - - profile.goroutine { - enabled = true - } - - profile.process_cpu { - enabled = true - } + arguments { + forward_to = [module.file.docker_compose.exports.profiles_receiver] + clustering = true } - - forward_to = [module.file.docker_compose.exports.profiles_receiver] } diff --git a/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml b/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml index 34d2f9fb..1f70dfc0 100644 --- a/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml +++ b/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml @@ -85,8 +85,13 @@ services: rules load agent-flow-mixin-alerts.yaml /loki-mixin-rules.yaml /loki-mixin-alerts.yaml /memcached-mixin-alerts.yaml /mimir-mixin-rules.yaml /mimir-mixin-alerts.yaml /pyroscope-mixin-rules.yaml loki: + # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md labels: - logs.agent.grafana.com/log-format=json + - profiles.agent.grafana.com/cpu.scrape=true + - profiles.agent.grafana.com/memory.scrape=true + - profiles.agent.grafana.com/goroutine.scrape=true + - profiles.agent.grafana.com/service_name=loki depends_on: minio: condition: service_healthy @@ -116,8 +121,13 @@ services: - loki-memberlist tempo: + # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md labels: - logs.agent.grafana.com/log-format=logfmt + - profiles.agent.grafana.com/cpu.scrape=true + - profiles.agent.grafana.com/memory.scrape=true + - profiles.agent.grafana.com/goroutine.scrape=true + - profiles.agent.grafana.com/service_name=tempo depends_on: minio: condition: service_healthy diff --git a/docker-compose/monolithic-mode/profiles/docker-compose.yaml b/docker-compose/monolithic-mode/profiles/docker-compose.yaml index 5b8bb109..69b1065a 100644 --- a/docker-compose/monolithic-mode/profiles/docker-compose.yaml +++ b/docker-compose/monolithic-mode/profiles/docker-compose.yaml @@ -54,6 +54,14 @@ services: retries: 5 grafana: + # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md + labels: + - logs.agent.grafana.com/log-format=logfmt + - profiles.agent.grafana.com/cpu.scrape=true + - profiles.agent.grafana.com/memory.scrape=true + - profiles.agent.grafana.com/goroutine.scrape=true + - profiles.agent.grafana.com/service_name=grafana + - profiles.agent.grafana.com/port=6060 image: ${GRAFANA_IMAGE:-docker.io/grafana/grafana:latest} hostname: grafana command: