From 039f9d2785d7e061bd1bdeec6c9ff0c706bde15d Mon Sep 17 00:00:00 2001 From: Weifeng Wang Date: Mon, 8 Apr 2024 16:14:58 +0800 Subject: [PATCH] New module for Grafana LGTMP Stack receiver provider Signed-off-by: Weifeng Wang Update provider.river --- docker-compose/common/config/.env | 2 +- .../common/config/agent-flow/.gitignore | 8 - .../common/config/agent-flow/logs.river | 15 +- .../common/config/agent-flow/metrics.river | 12 +- .../config/agent-flow/modules/README.md | 52 ++ .../docker/compose/logs-auto-scrape.river | 178 +++++ .../docker/compose/metrics-auto-scrape.river | 406 +++++++++++ .../docker/compose/profiles-auto-scrape.river | 653 ++++++++++++++++++ .../agent-flow/modules/docker/logs/all.river | 14 +- .../docker/logs/logs-auto-scrape.river | 171 ----- .../modules/docker/metrics/all.river | 16 +- .../jobs/integration_node_exporter.river | 2 +- .../docker/metrics/label-auto-scrape.river | 399 ----------- .../modules/docker/profiles/all.river | 12 +- .../profiles/profiles-auto-scrape.river | 646 ----------------- .../agent-flow/modules/grafana_cloud.river | 114 --- .../config/agent-flow/modules/local.river | 95 --- .../config/agent-flow/modules/provider.river | 194 ++++++ .../agent-flow/monolithic-mode-all.river | 40 +- .../common/config/agent-flow/profiles.river | 15 +- .../common/config/agent-flow/traces.river | 21 +- 21 files changed, 1552 insertions(+), 1513 deletions(-) delete mode 100644 docker-compose/common/config/agent-flow/.gitignore create mode 100644 docker-compose/common/config/agent-flow/modules/README.md create mode 100644 docker-compose/common/config/agent-flow/modules/docker/compose/logs-auto-scrape.river create mode 100644 docker-compose/common/config/agent-flow/modules/docker/compose/metrics-auto-scrape.river create mode 100644 docker-compose/common/config/agent-flow/modules/docker/compose/profiles-auto-scrape.river delete mode 100644 docker-compose/common/config/agent-flow/modules/docker/logs/logs-auto-scrape.river delete mode 100644 docker-compose/common/config/agent-flow/modules/docker/metrics/label-auto-scrape.river delete mode 100644 docker-compose/common/config/agent-flow/modules/docker/profiles/profiles-auto-scrape.river delete mode 100644 docker-compose/common/config/agent-flow/modules/grafana_cloud.river delete mode 100644 docker-compose/common/config/agent-flow/modules/local.river create mode 100644 docker-compose/common/config/agent-flow/modules/provider.river diff --git a/docker-compose/common/config/.env b/docker-compose/common/config/.env index 24065d97..ae620f50 100644 --- a/docker-compose/common/config/.env +++ b/docker-compose/common/config/.env @@ -1,6 +1,6 @@ AGENT_IMAGE=grafana/agent:v0.40.3 LOKI_IMAGE=grafana/loki:2.9.6 -GRAFANA_IMAGE=grafana/grafana:10.4.1 +GRAFANA_IMAGE=grafana/grafana:11.0.0-preview TEMPO_IMAGE=grafana/tempo:2.4.1 MIMIR_IMAGE=grafana/mimir:2.12.0 MIMIRTOOL_IMAGE=grafana/mimirtool:2.12.0 diff --git a/docker-compose/common/config/agent-flow/.gitignore b/docker-compose/common/config/agent-flow/.gitignore deleted file mode 100644 index db6a3c8b..00000000 --- a/docker-compose/common/config/agent-flow/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -* -!.gitignore -!modules -!logs.river -!metrics.river -!monolithic-mode-all.river -!profiles.river -!traces.river diff --git a/docker-compose/common/config/agent-flow/logs.river b/docker-compose/common/config/agent-flow/logs.river index 1fe0584d..c4e97877 100644 --- a/docker-compose/common/config/agent-flow/logs.river +++ b/docker-compose/common/config/agent-flow/logs.river @@ -8,12 +8,13 @@ logging { /******************************************** * Grafana LGTMP Stack Receiver Provider ********************************************/ -module.file "lgtmp_provider_local" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/local.river" +import.file "provider" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/provider.river" +} - arguments { - logs_endpoint = "http://gateway:3100" - } +provider.lgtmp_stack "docker_compose" { + logs_endpoint = "http://gateway:3100" + metrics_endpoint = "http://gateway:8080" } /******************************************** @@ -23,7 +24,7 @@ module.file "logs_primary" { filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/all.river" arguments { - forward_to = [module.file.lgtmp_provider_local.exports.logs_receiver] + forward_to = [provider.lgtmp_stack.docker_compose.logs_receiver] } } @@ -34,6 +35,6 @@ module.file "metrics_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river" arguments { - forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver] + forward_to = [provider.lgtmp_stack.docker_compose.metrics_receiver] } } diff --git a/docker-compose/common/config/agent-flow/metrics.river b/docker-compose/common/config/agent-flow/metrics.river index 5ced4580..3a392bf3 100644 --- a/docker-compose/common/config/agent-flow/metrics.river +++ b/docker-compose/common/config/agent-flow/metrics.river @@ -8,12 +8,12 @@ logging { /******************************************** * Grafana LGTMP Stack Receiver Provider ********************************************/ -module.file "lgtmp_provider_local" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/local.river" +import.file "provider" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/provider.river" +} - arguments { - metrics_endpoint = "http://gateway:8080" - } +provider.lgtmp_stack "docker_compose" { + metrics_endpoint = "http://gateway:8080" } /******************************************** @@ -23,6 +23,6 @@ module.file "metrics_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river" arguments { - forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver] + forward_to = [provider.lgtmp_stack.docker_compose.metrics_receiver] } } diff --git a/docker-compose/common/config/agent-flow/modules/README.md b/docker-compose/common/config/agent-flow/modules/README.md new file mode 100644 index 00000000..bcdf995c --- /dev/null +++ b/docker-compose/common/config/agent-flow/modules/README.md @@ -0,0 +1,52 @@ +# LGTMP agent moudles + + + +```river +import.git "provider" { + repository = "https://github.com/qclaogui/codelab-monitoring.git" + revision = "main" + path = "docker-compose/common/config/agent-flow/modules/provider.river" + pull_frequency = "15m" +} + +// get the receivers provider +provider.lgtmp_stack "docker_compose" { + metrics_endpoint = "http://mimir:8080" +} + +// scrape metrics and write to grafana cloud +prometheus.scrape "default" { + targets = [ + {"__address__" = "127.0.0.1:12345"}, + ] + forward_to = [ + provider.lgtmp_stack.docker_compose.metrics_receiver, + ] +} +``` + +```river +import.git "provider" { + repository = "https://github.com/qclaogui/codelab-monitoring.git" + revision = "main" + path = "docker-compose/common/config/agent-flow/modules/provider.river" + pull_frequency = "15m" +} + +// get the receivers provider +provider.grafana_cloud "stack_name" { + stack_name = env("GRAFANA_CLOUD_STACK_NAME") + token = env("GRAFANA_CLOUD_STACK_TOKEN") +} + +// scrape metrics and write to grafana cloud +prometheus.scrape "default" { + targets = [ + {"__address__" = "127.0.0.1:12345"}, + ] + forward_to = [ + provider.grafana_cloud.stack_name.metrics_receiver, + ] +} +``` diff --git a/docker-compose/common/config/agent-flow/modules/docker/compose/logs-auto-scrape.river b/docker-compose/common/config/agent-flow/modules/docker/compose/logs-auto-scrape.river new file mode 100644 index 00000000..3e7ec0ce --- /dev/null +++ b/docker-compose/common/config/agent-flow/modules/docker/compose/logs-auto-scrape.river @@ -0,0 +1,178 @@ +/* +Module(logs_auto_scrape): Docker Containers Logs Auto-Scraping +Description: Scrapes targets for logs based on Docker Containers labels + +Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. + + +Following labels are available: + + logs.agent.grafana.com/scrape: true + +allow containers to declare what tenant their logs should be written to, +the following label is supported: + + logs.agent.grafana.com/tenant: "primary" +*/ + +declare "logs_auto_scrape" { + + /******************************************** + * ARGUMENTS + ********************************************/ + argument "forward_to" { + comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + } + + argument "targets" { + comment = "The running containers." + } + + argument "cluster" { + optional = true + } + + argument "namespace" { + optional = true + } + + argument "tenant" { + comment = "The tenant to write metrics to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant label, and this can be a regex. (default: (.*))" + optional = true + } + + // get logs from discovery relabel dr_docker_logs below + loki.source.docker "lsd_docker_logs" { + forward_to = argument.forward_to.value + + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.dr_docker_logs.output + relabel_rules = discovery.relabel.dr_docker_logs.rules + refresh_interval = "15s" + } + + discovery.relabel "dr_docker_logs" { + targets = argument.targets.value + + /**************************************************************************************************************** + * Handle Discovers From Docker Engine Containers Targets to Keep or Drop + * https://grafana.com/docs/agent/latest/flow/reference/components/discovery.docker/#exported-fields + ****************************************************************************************************************/ + // allow resources to declare their metrics scraped or not + // Example Annotation: + // logs.agent.grafana.com/scrape: false + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_logs_agent_grafana_com_scrape", + ] + separator = ";" + regex = "^(?:;*)?(true|false).*$" + replacement = "$1" + target_label = "__tmp_scrape" + } + + // drop any targets that have scrape: false + rule { + action = "drop" + source_labels = ["__tmp_scrape"] + regex = "false" + } + + // allow resources to declare their metrics the tenant their metrics should be sent to, + // Example Annotation: + // logs.agent.grafana.com/tenant: primary + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_logs_agent_grafana_com_tenant", + ] + regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" + } + + // make all labels on the pod available to the pipeline as labels(for loki process), + // they are omitted before write via labelallow unless explicitly set + rule { + action = "labelmap" + regex = "__meta_docker_container_label_(.+)" + } + + /******************************************** + * Handle Setting Common Labels + ********************************************/ + + // set the cluster label + rule { + action = "replace" + replacement = coalesce(argument.cluster.value, "docker-compose") + target_label = "cluster" + } + + // set the namespace label + rule { + action = "replace" + replacement = coalesce(argument.namespace.value, "monitoring-system") + target_label = "namespace" + } + + // set a default job label to be the namespace/service_name + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = coalesce(argument.namespace.value, "monitoring-system") + "/$1" + target_label = "job" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "pod" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "container" + } + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container_name" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_app", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + } +} diff --git a/docker-compose/common/config/agent-flow/modules/docker/compose/metrics-auto-scrape.river b/docker-compose/common/config/agent-flow/modules/docker/compose/metrics-auto-scrape.river new file mode 100644 index 00000000..92730a93 --- /dev/null +++ b/docker-compose/common/config/agent-flow/modules/docker/compose/metrics-auto-scrape.river @@ -0,0 +1,406 @@ +/* +Module(metrics): Docker Containers Auto-Scraping +Description: Scrapes targets for metrics based on Docker Containers labels + +Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. + + +Following labels are available: + + metrics.agent.grafana.com/scrape: true + +the default scraping scheme is http, only support http now. + + metrics.agent.grafana.com/scheme: http +or + prometheus.io/scheme: http + +the default path to scrape is /metrics, this can be specified as a single value which would override, the scrape path being used +for all ports attached to the target: + + metrics.agent.grafana.com/path: /metrics/some_path + +the default port to scrape is the target port, this can be specified as a single value which would override the scrape port being +used for all ports attached to the target, note that even if an target had multiple targets, the relabel_config targets are +deduped before scraping: + + metrics.agent.grafana.com/port: 8080 +or + prometheus.io/port: 8080 + +the default interval to scrape is 15s, this can be specified as a single value which would override, the scrape interval being used +for all ports attached to the target: + + metrics.agent.grafana.com/interval: 15s +or + prometheus.io/interval: 15s + + +the default timeout for scraping is 10s, this can be specified as a single value which would override, the scrape interval being +used for all ports attached to the target: + + metrics.agent.grafana.com/timeout: 10s +or + prometheus.io/timeout: 10s + +the default job is namespace/{{ service name }} there may be a different job name is required because of a set of dashboards, rules, +etc. to support this there is a job annotation which will override the default value: + + metrics.agent.grafana.com/job: integrations/kubernetes/kube-state-metrics +or + prometheus.io/job: integrations/kubernetes/kube-state-metrics +*/ + +declare "metrics_auto_scrape" { + + /******************************************** + * ARGUMENTS + ********************************************/ + argument "forward_to" { + comment = "Must be a list(MetricsReceiver) where collected metrics should be forwarded to" + } + + argument "targets" { + comment = "The running containers." + } + + argument "cluster" { + optional = true + } + + argument "namespace" { + optional = true + } + + argument "tenant" { + comment = "The tenant to write metrics to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant label, and this can be a regex. (default: (.*))" + optional = true + } + + argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true + } + + argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true + } + + argument "scrape_interval" { + comment = "How often to scrape metrics from the targets (default: 60s)" + optional = true + } + + argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 10s)" + optional = true + } + + /******************************************** + * Discovery Relabelings (pre-scrape) + ********************************************/ + discovery.relabel "dr_label_metrics" { + targets = argument.targets.value + + /**************************************************************************************************************** + * Handle Discovers From Docker Engine Containers Targets to Keep or Drop + * https://grafana.com/docs/agent/latest/flow/reference/components/discovery.docker/#exported-fields + ****************************************************************************************************************/ + // allow resources to declare their metrics scraped or not + // Example Annotation: + // metrics.agent.grafana.com/scrape: false + // + // the label prometheus.io/service-monitor: "false" is a common label for headless services, when performing endpoint + // service discovery, if there is both a load-balanced service and headless service, this can result in duplicate + // scrapes if the name of the service is attached as a label. any targets with this label or annotation set should be dropped + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_metrics_agent_grafana_com_scrape", + "__meta_docker_container_label_prometheus_io_scrape", + ] + separator = ";" + regex = "^(?:;*)?(true|false).*$" + replacement = "$1" + target_label = "__tmp_scrape" + } + + // drop any targets that have scrape: false + rule { + action = "drop" + source_labels = ["__tmp_scrape"] + regex = "false" + } + + // allow resources to declare the protocol to use when collecting metrics, the default value is "http", + // Example Annotation: + // metrics.agent.grafana.com/scheme: http + rule { + action = "replace" + replacement = "http" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_metrics_agent_grafana_com_scheme", + "__meta_docker_container_label_prometheus_io_scheme", + ] + separator = ";" + regex = "^(?:;*)?(https?).*$" + replacement = "$1" + target_label = "__scheme__" + } + + // allow resources to declare their metrics the tenant their metrics should be sent to, + // Example Annotation: + // metrics.agent.grafana.com/tenant: primary + // + // Note: This does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used + // to determine if the metrics should be gathered for the current tenant + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_metrics_agent_grafana_com_tenant", + "__meta_docker_container_label_prometheus_io_tenant", + ] + regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_metrics_agent_grafana_com_port", + "__meta_docker_container_label_prometheus_io_port", + ] + separator = ";" + regex = "^(?:;*)?(\\d+).*$" + replacement = "$1" + target_label = "__tmp_metrics_port" + } + + // allow resources to declare the port to use when collecting metrics, the default value is the discovered port from + // Example Annotation: + // metrics.agent.grafana.com/port: 9090 + rule { + action = "replace" + source_labels = [ + "__address__", + "__tmp_metrics_port", + ] + separator = ";" + regex = "^([^:]+)(?::\\d+)?;(\\d+)$" + replacement = "$1:$2" + target_label = "__address__" + } + + // allow resources to declare their the path to use when collecting their metrics, the default value is "/metrics", + // Example Annotation: + // metrics.agent.grafana.com/path: /metrics/foo + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_metrics_agent_grafana_com_path", + "__meta_docker_container_label_prometheus_io_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__metrics_path__" + } + + // allow resources to declare how often their metrics should be collected, the default value is 15s, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // metrics.agent.grafana.com/interval: 15s + rule { + action = "replace" + replacement = coalesce(argument.scrape_interval.value, "60s") + target_label = "__scrape_interval__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_metrics_agent_grafana_com_interval", + "__meta_docker_container_label_prometheus_io_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_interval__" + } + + // allow resources to declare the timeout of the scrape request, the default value is 10s, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // metrics.agent.grafana.com/timeout: 10s + rule { + action = "replace" + replacement = coalesce(argument.scrape_timeout.value, "10s") + target_label = "__scrape_timeout__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_metrics_agent_grafana_com_interval", + "__meta_docker_container_label_prometheus_io_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_timeout__" + } + + /******************************************** + * Handle Setting Common Labels + ********************************************/ + + // set the cluster label + rule { + action = "replace" + replacement = coalesce(argument.cluster.value, "docker-compose") + target_label = "cluster" + } + + // set the namespace label + rule { + action = "replace" + replacement = coalesce(argument.namespace.value, "monitoring-system") + target_label = "namespace" + } + + // set a default job label to be the namespace/service_name + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = coalesce(argument.namespace.value, "monitoring-system") + "/$1" + target_label = "job" + } + + // allow resources to declare their the job label value to use when collecting their metrics, the default value is "", + // Example Annotation: + // metrics.agent.grafana.com/job: integrations/kubernetes/cadvisor + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_metrics_agent_grafana_com_job", + "__meta_docker_container_label_prometheus_io_job", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "job" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "pod" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "container" + } + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container_name" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_app", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + } + + // only keep http targets + discovery.relabel "dr_keep_http_targets" { + targets = discovery.relabel.dr_label_metrics.output + + rule { + action = "keep" + source_labels = ["__scheme__"] + regex = "http" + } + } + + /******************************************** + * Prometheus Scrape Labels Targets + ********************************************/ + prometheus.scrape "pc_docker_metrics" { + targets = concat( + discovery.relabel.dr_keep_http_targets.output, + ) + + job_name = "label-metrics" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + + enable_protobuf_negotiation = true + scrape_classic_histograms = true + + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.pr_docker_metrics.receiver] + } + + /******************************************** + * Prometheus Metric Relabelings (post-scrape) + ********************************************/ + // perform generic relabeling using keep_metrics and drop_metrics + prometheus.relabel "pr_docker_metrics" { + forward_to = argument.forward_to.value + + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } + } +} diff --git a/docker-compose/common/config/agent-flow/modules/docker/compose/profiles-auto-scrape.river b/docker-compose/common/config/agent-flow/modules/docker/compose/profiles-auto-scrape.river new file mode 100644 index 00000000..a12d3853 --- /dev/null +++ b/docker-compose/common/config/agent-flow/modules/docker/compose/profiles-auto-scrape.river @@ -0,0 +1,653 @@ +/* +Module(profiles_auto_scrape): Docker Containers Auto-Scraping +Description: Scrapes targets for profiles based on Docker Containers labels + +Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. + + +The full list of profile types supported by labels is "cpu", "memory", "goroutine", "block", "mutex" and "fgprof" + +For example the following labels: + + ``` + profiles.agent.grafana.com/.scrape: true + profiles.agent.grafana.com/.path: /path/to + profiles.agent.grafana.com/service_name: pyroscope + profiles.agent.grafana.com/tenant: primary + profiles.agent.grafana.com/scheme: http + profiles.agent.grafana.com/port: 8080 + profiles.agent.grafana.com/interval: 30s + profiles.agent.grafana.com/timeout: 15s +or + profiles.agent.grafana.com/.scrape: true + profiles.agent.grafana.com/.path: /path/to + profiles.agent.grafana.com/service_name: pyroscope + profiles.agent.grafana.com/tenant: primary + profiles.agent.grafana.com/scheme: http + profiles.agent.grafana.com/port: 8080 + profiles.agent.grafana.com/interval: 30s + profiles.agent.grafana.com/timeout: 15s + ``` +*/ + +declare "profiles_auto_scrape" { + + /******************************************** + * ARGUMENTS + ********************************************/ + argument "forward_to" { + comment = "Must be a list(ProfilessReceiver) where collected profile should be forwarded to" + } + + argument "cluster" { + optional = true + } + + argument "namespace" { + optional = true + } + + argument "tenant" { + comment = "The tenant to write profile to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant label, and this can be a regex. (default: (.*))" + optional = true + } + + argument "scrape_interval" { + comment = "How often to scrape profile from the targets (default: 30s)" + optional = true + } + + argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 15s)" + optional = true + } + + // get the available containers. + discovery.docker "dd_profiles" { + host = "unix:///var/run/docker.sock" + refresh_interval = "15s" + + filter { + name = "status" + values = ["running"] + } + } + + discovery.relabel "dr_docker_profiles" { + targets = discovery.docker.dd_profiles.targets + + // allow resources to declare the protocol to use when collecting profiles + // default value is "http" + rule { + action = "replace" + replacement = "http" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_scheme", + "__meta_docker_container_label_profiles_grafana_com_scheme", + ] + separator = ";" + regex = "^(?:;*)?(https?).*$" + replacement = "$1" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_port", + "__meta_docker_container_label_profiles_grafana_com_port", + ] + separator = ";" + regex = "^(?:;*)?(\\d+).*$" + replacement = "$1" + target_label = "__tmp_port" + } + + rule { + action = "replace" + source_labels = [ + "__address__", + "__tmp_port", + ] + separator = ";" + regex = "^([^:]+)(?::\\d+)?;(\\d+)$" + replacement = "$1:$2" + target_label = "__address__" + } + + // allow resources to declare their profiles should be sent to + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_tenant", + "__meta_docker_container_label_profiles_grafana_com_tenant", + ] + regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" + } + + // allow resources to declare how often their profiles should be collected. + // default value is 30s, the following duration formats are supported (s|m|ms|h|d) + rule { + action = "replace" + replacement = coalesce(argument.scrape_interval.value, "30s") + target_label = "__scrape_interval__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_interval", + "__meta_docker_container_label_profiles_grafana_com_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_interval__" + } + + // allow resources to declare the timeout of the scrape request + // default value is 15s, the following duration formats are supported (s|m|ms|h|d) + rule { + action = "replace" + replacement = coalesce(argument.scrape_timeout.value, "15s") + target_label = "__scrape_timeout__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_interval", + "__meta_docker_container_label_profiles_grafana_com_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_timeout__" + } + + /******************************************** + * Handle Setting Common Labels + ********************************************/ + + // set the cluster label + rule { + action = "replace" + replacement = coalesce(argument.cluster.value, "docker-compose") + target_label = "cluster" + } + + // set the namespace label + rule { + action = "replace" + replacement = coalesce(argument.namespace.value, "monitoring-system") + target_label = "namespace" + } + + // set a default job label to be the namespace/docker_compose_service + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = coalesce(argument.namespace.value, "monitoring-system") + "/$1" + target_label = "job" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "pod" + } + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_app", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_com_docker_compose_service", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "service_name" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_service_name", + "__meta_docker_container_label_profiles_grafana_com_service_name", + "__meta_docker_container_label_pyroscope_io_service_name", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "service_name" + } + } + + /******************************************** + * Pyroscope Scrape CPU + ********************************************/ + discovery.relabel "dr_keep_cpu_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_cpu_scrape", + "__meta_docker_container_label_profiles_grafana_com_cpu_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_cpu_path", + "__meta_docker_container_label_profiles_grafana_com_cpu_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } + } + + pyroscope.scrape "ps_profile_cpu" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-cpu" + targets = discovery.relabel.dr_keep_cpu_targets.output + + clustering { + enabled = true + } + + profiling_config { + profile.process_cpu { + enabled = true + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = false + } + } + } + + /******************************************** + * Pyroscope Scrape Memory + ********************************************/ + discovery.relabel "dr_keep_memory_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_memory_scrape", + "__meta_docker_container_label_profiles_grafana_com_memory_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_memory_path", + "__meta_docker_container_label_profiles_grafana_com_memory_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } + } + + pyroscope.scrape "ps_profile_memory" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-memory" + targets = discovery.relabel.dr_keep_memory_targets.output + + clustering { + enabled = true + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = true + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = false + } + } + } + + /******************************************** + * Pyroscope Scrape Goroutine + ********************************************/ + discovery.relabel "dr_keep_goroutine_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_goroutine_scrape", + "__meta_docker_container_label_profiles_grafana_com_goroutine_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_goroutine_path", + "__meta_docker_container_label_profiles_grafana_com_goroutine_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } + } + + pyroscope.scrape "ps_profile_goroutine" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-goroutine" + targets = discovery.relabel.dr_keep_goroutine_targets.output + + clustering { + enabled = true + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = true + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = false + } + } + } + + /******************************************** + * Pyroscope Scrape Block + ********************************************/ + discovery.relabel "dr_keep_block_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_block_scrape", + "__meta_docker_container_label_profiles_grafana_com_block_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_block_path", + "__meta_docker_container_label_profiles_grafana_com_block_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } + } + + pyroscope.scrape "ps_profile_block" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-block" + targets = discovery.relabel.dr_keep_block_targets.output + + clustering { + enabled = true + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = true + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = false + } + } + } + + /******************************************** + * Pyroscope Scrape Mutex + ********************************************/ + discovery.relabel "dr_keep_mutex_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_mutex_scrape", + "__meta_docker_container_label_profiles_grafana_com_mutex_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_mutex_path", + "__meta_docker_container_label_profiles_grafana_com_mutex_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } + } + + pyroscope.scrape "ps_profile_mutex" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-mutex" + targets = discovery.relabel.dr_keep_mutex_targets.output + + clustering { + enabled = true + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = true + } + + profile.fgprof { + enabled = false + } + } + } + + /******************************************** + * Pyroscope Scrape Fgprof + ********************************************/ + discovery.relabel "dr_keep_fgprof_targets" { + targets = discovery.relabel.dr_docker_profiles.output + + rule { + action = "keep" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_fgprof_scrape", + "__meta_docker_container_label_profiles_grafana_com_fgprof_scrape", + ] + regex = "^(?:;*)?(true).*$" + } + + rule { + action = "replace" + source_labels = [ + "__meta_docker_container_label_profiles_agent_grafana_com_fgprof_path", + "__meta_docker_container_label_profiles_grafana_com_fgprof_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__profile_path__" + } + } + + pyroscope.scrape "ps_profile_fgprof" { + forward_to = argument.forward_to.value + + job_name = "label-profiles-fgprof" + targets = discovery.relabel.dr_keep_fgprof_targets.output + + clustering { + enabled = true + } + + profiling_config { + profile.process_cpu { + enabled = false + } + + profile.memory { + enabled = false + } + + profile.goroutine { + enabled = false + } + + profile.block { + enabled = false + } + + profile.mutex { + enabled = false + } + + profile.fgprof { + enabled = true + } + } + } +} diff --git a/docker-compose/common/config/agent-flow/modules/docker/logs/all.river b/docker-compose/common/config/agent-flow/modules/docker/logs/all.river index aa8c4cd4..ada4812e 100644 --- a/docker-compose/common/config/agent-flow/modules/docker/logs/all.river +++ b/docker-compose/common/config/agent-flow/modules/docker/logs/all.river @@ -47,14 +47,14 @@ discovery.docker "dd_logs" { } } -module.file "mf_logs_auto_scrape" { - filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/logs-auto-scrape.river" +import.file "docker_compose" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/compose/logs-auto-scrape.river" +} - arguments { - targets = discovery.docker.dd_logs.targets - forward_to = [module.file.mf_log_formats_all.exports.process.receiver] - tenant = argument.tenant.value - } +docker_compose.logs_auto_scrape "default" { + targets = discovery.docker.dd_logs.targets + forward_to = [module.file.mf_log_formats_all.exports.process.receiver] + tenant = coalesce(argument.tenant.value, ".*") } module.file "mf_log_formats_all" { diff --git a/docker-compose/common/config/agent-flow/modules/docker/logs/logs-auto-scrape.river b/docker-compose/common/config/agent-flow/modules/docker/logs/logs-auto-scrape.river deleted file mode 100644 index 171f6f09..00000000 --- a/docker-compose/common/config/agent-flow/modules/docker/logs/logs-auto-scrape.river +++ /dev/null @@ -1,171 +0,0 @@ -/* -Module(logs): Docker Containers Logs Auto-Scraping -Description: Scrapes targets for logs based on Docker Containers labels - -Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these - arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). - This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will - does not override the value passed in, where coalesce() will return the first non-null value. - - -Following labels are available: - - logs.agent.grafana.com/scrape: true - -allow containers to declare what tenant their logs should be written to, -the following label is supported: - - logs.agent.grafana.com/tenant: "primary" -*/ -argument "forward_to" { - comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" -} - -argument "targets" { - comment = "The running containers." -} - -argument "cluster" { - optional = true -} - -argument "namespace" { - optional = true -} - -argument "tenant" { - comment = "The tenant to write metrics to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant label, and this can be a regex. (default: (.*))" - optional = true -} - -// get logs from discovery relabel dr_docker_logs below -loki.source.docker "lsd_docker_logs" { - forward_to = argument.forward_to.value - - host = "unix:///var/run/docker.sock" - targets = discovery.relabel.dr_docker_logs.output - relabel_rules = discovery.relabel.dr_docker_logs.rules - refresh_interval = "15s" -} - -discovery.relabel "dr_docker_logs" { - targets = argument.targets.value - - /**************************************************************************************************************** - * Handle Discovers From Docker Engine Containers Targets to Keep or Drop - * https://grafana.com/docs/agent/latest/flow/reference/components/discovery.docker/#exported-fields - ****************************************************************************************************************/ - // allow resources to declare their metrics scraped or not - // Example Annotation: - // logs.agent.grafana.com/scrape: false - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_logs_agent_grafana_com_scrape", - ] - separator = ";" - regex = "^(?:;*)?(true|false).*$" - replacement = "$1" - target_label = "__tmp_scrape" - } - - // drop any targets that have scrape: false - rule { - action = "drop" - source_labels = ["__tmp_scrape"] - regex = "false" - } - - // allow resources to declare their metrics the tenant their metrics should be sent to, - // Example Annotation: - // logs.agent.grafana.com/tenant: primary - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_logs_agent_grafana_com_tenant", - ] - regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" - } - - // make all labels on the pod available to the pipeline as labels(for loki process), - // they are omitted before write via labelallow unless explicitly set - rule { - action = "labelmap" - regex = "__meta_docker_container_label_(.+)" - } - - /******************************************** - * Handle Setting Common Labels - ********************************************/ - - // set the cluster label - rule { - action = "replace" - replacement = coalesce(argument.cluster.value, "docker-compose") - target_label = "cluster" - } - - // set the namespace label - rule { - action = "replace" - replacement = coalesce(argument.namespace.value, "monitoring-system") - target_label = "namespace" - } - - // set a default job label to be the namespace/service_name - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = coalesce(argument.namespace.value, "monitoring-system") + "/$1" - target_label = "job" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "pod" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "container" - } - - rule { - source_labels = ["__meta_docker_container_name"] - regex = "/(.*)" - target_label = "container_name" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "app" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_app", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "app" - } -} diff --git a/docker-compose/common/config/agent-flow/modules/docker/metrics/all.river b/docker-compose/common/config/agent-flow/modules/docker/metrics/all.river index c18ca86f..4ccb1b78 100644 --- a/docker-compose/common/config/agent-flow/modules/docker/metrics/all.river +++ b/docker-compose/common/config/agent-flow/modules/docker/metrics/all.river @@ -22,15 +22,15 @@ discovery.docker "dd_metrics" { } } -module.file "mf_label_auto_scrape" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/label-auto-scrape.river" +import.file "docker_compose" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/compose/metrics-auto-scrape.river" +} - arguments { - targets = discovery.docker.dd_metrics.targets - forward_to = argument.forward_to.value - tenant = coalesce(argument.tenant.value, ".*") - scrape_interval = "15s" - } +docker_compose.metrics_auto_scrape "default" { + targets = discovery.docker.dd_metrics.targets + forward_to = argument.forward_to.value + tenant = coalesce(argument.tenant.value, ".*") + scrape_interval = "15s" } module.file "mf_job_minio_scrape" { diff --git a/docker-compose/common/config/agent-flow/modules/docker/metrics/jobs/integration_node_exporter.river b/docker-compose/common/config/agent-flow/modules/docker/metrics/jobs/integration_node_exporter.river index c5ccff5f..ce826a4f 100644 --- a/docker-compose/common/config/agent-flow/modules/docker/metrics/jobs/integration_node_exporter.river +++ b/docker-compose/common/config/agent-flow/modules/docker/metrics/jobs/integration_node_exporter.river @@ -63,7 +63,7 @@ prometheus.relabel "pr_node_exporter" { // keep only metrics that match the keep_metrics regex rule { source_labels = ["__name__"] - regex = coalesce(argument.keep_metrics.value, "(up|node_exporter_build_info|node_cpu.*|node_memory.*|node_disk.*|node_filesystem.*|process_cpu_seconds_total|process_resident_memory_bytes)") + regex = coalesce(argument.keep_metrics.value, "(up|node_exporter_build_info|node_time_seconds|node_boot_time_seconds|node_load.*|node_cpu.*|node_memory.*|node_disk.*|node_filesystem.*|process_cpu_seconds_total|process_resident_memory_bytes)") action = "keep" } diff --git a/docker-compose/common/config/agent-flow/modules/docker/metrics/label-auto-scrape.river b/docker-compose/common/config/agent-flow/modules/docker/metrics/label-auto-scrape.river deleted file mode 100644 index 692d6e1b..00000000 --- a/docker-compose/common/config/agent-flow/modules/docker/metrics/label-auto-scrape.river +++ /dev/null @@ -1,399 +0,0 @@ -/* -Module(metrics): Docker Containers Auto-Scraping -Description: Scrapes targets for metrics based on Docker Containers labels - -Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these - arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). - This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will - does not override the value passed in, where coalesce() will return the first non-null value. - - -Following labels are available: - - metrics.agent.grafana.com/scrape: true - -the default scraping scheme is http, only support http now. - - metrics.agent.grafana.com/scheme: http -or - prometheus.io/scheme: http - -the default path to scrape is /metrics, this can be specified as a single value which would override, the scrape path being used -for all ports attached to the target: - - metrics.agent.grafana.com/path: /metrics/some_path - -the default port to scrape is the target port, this can be specified as a single value which would override the scrape port being -used for all ports attached to the target, note that even if an target had multiple targets, the relabel_config targets are -deduped before scraping: - - metrics.agent.grafana.com/port: 8080 -or - prometheus.io/port: 8080 - -the default interval to scrape is 15s, this can be specified as a single value which would override, the scrape interval being used -for all ports attached to the target: - - metrics.agent.grafana.com/interval: 15s -or - prometheus.io/interval: 15s - - -the default timeout for scraping is 10s, this can be specified as a single value which would override, the scrape interval being -used for all ports attached to the target: - - metrics.agent.grafana.com/timeout: 10s -or - prometheus.io/timeout: 10s - -the default job is namespace/{{ service name }} there may be a different job name is required because of a set of dashboards, rules, -etc. to support this there is a job annotation which will override the default value: - - metrics.agent.grafana.com/job: integrations/kubernetes/kube-state-metrics -or - prometheus.io/job: integrations/kubernetes/kube-state-metrics -*/ -argument "forward_to" { - comment = "Must be a list(MetricsReceiver) where collected metrics should be forwarded to" -} - -argument "targets" { - comment = "The running containers." -} - -argument "cluster" { - optional = true -} - -argument "namespace" { - optional = true -} - -argument "tenant" { - comment = "The tenant to write metrics to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant label, and this can be a regex. (default: (.*))" - optional = true -} - -argument "keep_metrics" { - comment = "A regex of metrics to keep (default: (.+))" - optional = true -} - -argument "drop_metrics" { - comment = "A regex of metrics to drop (default: \"\")" - optional = true -} - -argument "scrape_interval" { - comment = "How often to scrape metrics from the targets (default: 60s)" - optional = true -} - -argument "scrape_timeout" { - comment = "How long before a scrape times out (default: 10s)" - optional = true -} - -/******************************************** - * Discovery Relabelings (pre-scrape) - ********************************************/ -discovery.relabel "dr_label_metrics" { - targets = argument.targets.value - - /**************************************************************************************************************** - * Handle Discovers From Docker Engine Containers Targets to Keep or Drop - * https://grafana.com/docs/agent/latest/flow/reference/components/discovery.docker/#exported-fields - ****************************************************************************************************************/ - // allow resources to declare their metrics scraped or not - // Example Annotation: - // metrics.agent.grafana.com/scrape: false - // - // the label prometheus.io/service-monitor: "false" is a common label for headless services, when performing endpoint - // service discovery, if there is both a load-balanced service and headless service, this can result in duplicate - // scrapes if the name of the service is attached as a label. any targets with this label or annotation set should be dropped - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_metrics_agent_grafana_com_scrape", - "__meta_docker_container_label_prometheus_io_scrape", - ] - separator = ";" - regex = "^(?:;*)?(true|false).*$" - replacement = "$1" - target_label = "__tmp_scrape" - } - - // drop any targets that have scrape: false - rule { - action = "drop" - source_labels = ["__tmp_scrape"] - regex = "false" - } - - // allow resources to declare the protocol to use when collecting metrics, the default value is "http", - // Example Annotation: - // metrics.agent.grafana.com/scheme: http - rule { - action = "replace" - replacement = "http" - target_label = "__scheme__" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_metrics_agent_grafana_com_scheme", - "__meta_docker_container_label_prometheus_io_scheme", - ] - separator = ";" - regex = "^(?:;*)?(https?).*$" - replacement = "$1" - target_label = "__scheme__" - } - - // allow resources to declare their metrics the tenant their metrics should be sent to, - // Example Annotation: - // metrics.agent.grafana.com/tenant: primary - // - // Note: This does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used - // to determine if the metrics should be gathered for the current tenant - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_metrics_agent_grafana_com_tenant", - "__meta_docker_container_label_prometheus_io_tenant", - ] - regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_metrics_agent_grafana_com_port", - "__meta_docker_container_label_prometheus_io_port", - ] - separator = ";" - regex = "^(?:;*)?(\\d+).*$" - replacement = "$1" - target_label = "__tmp_metrics_port" - } - - // allow resources to declare the port to use when collecting metrics, the default value is the discovered port from - // Example Annotation: - // metrics.agent.grafana.com/port: 9090 - rule { - action = "replace" - source_labels = [ - "__address__", - "__tmp_metrics_port", - ] - separator = ";" - regex = "^([^:]+)(?::\\d+)?;(\\d+)$" - replacement = "$1:$2" - target_label = "__address__" - } - - // allow resources to declare their the path to use when collecting their metrics, the default value is "/metrics", - // Example Annotation: - // metrics.agent.grafana.com/path: /metrics/foo - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_metrics_agent_grafana_com_path", - "__meta_docker_container_label_prometheus_io_path", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "__metrics_path__" - } - - // allow resources to declare how often their metrics should be collected, the default value is 15s, - // the following duration formats are supported (s|m|ms|h|d): - // Example Annotation: - // metrics.agent.grafana.com/interval: 15s - rule { - action = "replace" - replacement = coalesce(argument.scrape_interval.value, "60s") - target_label = "__scrape_interval__" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_metrics_agent_grafana_com_interval", - "__meta_docker_container_label_prometheus_io_interval", - ] - separator = ";" - regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" - replacement = "$1" - target_label = "__scrape_interval__" - } - - // allow resources to declare the timeout of the scrape request, the default value is 10s, - // the following duration formats are supported (s|m|ms|h|d): - // Example Annotation: - // metrics.agent.grafana.com/timeout: 10s - rule { - action = "replace" - replacement = coalesce(argument.scrape_timeout.value, "10s") - target_label = "__scrape_timeout__" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_metrics_agent_grafana_com_interval", - "__meta_docker_container_label_prometheus_io_interval", - ] - separator = ";" - regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" - replacement = "$1" - target_label = "__scrape_timeout__" - } - - /******************************************** - * Handle Setting Common Labels - ********************************************/ - - // set the cluster label - rule { - action = "replace" - replacement = coalesce(argument.cluster.value, "docker-compose") - target_label = "cluster" - } - - // set the namespace label - rule { - action = "replace" - replacement = coalesce(argument.namespace.value, "monitoring-system") - target_label = "namespace" - } - - // set a default job label to be the namespace/service_name - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = coalesce(argument.namespace.value, "monitoring-system") + "/$1" - target_label = "job" - } - - // allow resources to declare their the job label value to use when collecting their metrics, the default value is "", - // Example Annotation: - // metrics.agent.grafana.com/job: integrations/kubernetes/cadvisor - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_metrics_agent_grafana_com_job", - "__meta_docker_container_label_prometheus_io_job", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "job" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "pod" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "container" - } - - rule { - source_labels = ["__meta_docker_container_name"] - regex = "/(.*)" - target_label = "container_name" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "app" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_app", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "app" - } -} - -// only keep http targets -discovery.relabel "dr_keep_http_targets" { - targets = discovery.relabel.dr_label_metrics.output - - rule { - action = "keep" - source_labels = ["__scheme__"] - regex = "http" - } -} - -/******************************************** - * Prometheus Scrape Labels Targets - ********************************************/ -prometheus.scrape "pc_docker_metrics" { - targets = concat( - discovery.relabel.dr_keep_http_targets.output, - ) - - job_name = "label-metrics" - scrape_interval = coalesce(argument.scrape_interval.value, "60s") - scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") - - enable_protobuf_negotiation = true - scrape_classic_histograms = true - - clustering { - enabled = true - } - - forward_to = [prometheus.relabel.pr_docker_metrics.receiver] -} - -/******************************************** - * Prometheus Metric Relabelings (post-scrape) - ********************************************/ -// perform generic relabeling using keep_metrics and drop_metrics -prometheus.relabel "pr_docker_metrics" { - forward_to = argument.forward_to.value - - // keep only metrics that match the keep_metrics regex - rule { - action = "keep" - source_labels = ["__name__"] - regex = coalesce(argument.keep_metrics.value, "(.+)") - } - - // drop metrics that match the drop_metrics regex - rule { - action = "drop" - source_labels = ["__name__"] - regex = coalesce(argument.drop_metrics.value, "") - } -} diff --git a/docker-compose/common/config/agent-flow/modules/docker/profiles/all.river b/docker-compose/common/config/agent-flow/modules/docker/profiles/all.river index 35af0e0e..abcdffb3 100644 --- a/docker-compose/common/config/agent-flow/modules/docker/profiles/all.river +++ b/docker-compose/common/config/agent-flow/modules/docker/profiles/all.river @@ -11,11 +11,11 @@ argument "tenant" { optional = true } -module.file "mf_profiles_auto_scrape" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/profiles/profiles-auto-scrape.river" +import.file "docker_compose" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/compose/profiles-auto-scrape.river" +} - arguments { - forward_to = argument.forward_to.value - tenant = coalesce(argument.tenant.value, ".*") - } +docker_compose.profiles_auto_scrape "default" { + forward_to = argument.forward_to.value + tenant = coalesce(argument.tenant.value, ".*") } diff --git a/docker-compose/common/config/agent-flow/modules/docker/profiles/profiles-auto-scrape.river b/docker-compose/common/config/agent-flow/modules/docker/profiles/profiles-auto-scrape.river deleted file mode 100644 index 1dc9d8a1..00000000 --- a/docker-compose/common/config/agent-flow/modules/docker/profiles/profiles-auto-scrape.river +++ /dev/null @@ -1,646 +0,0 @@ -/* -Module(profiles): Docker Containers Auto-Scraping -Description: Scrapes targets for profiles based on Docker Containers labels - -Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these - arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). - This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will - does not override the value passed in, where coalesce() will return the first non-null value. - - -The full list of profile types supported by labels is "cpu", "memory", "goroutine", "block", "mutex" and "fgprof" - -For example the following labels: - - ``` - profiles.agent.grafana.com/.scrape: true - profiles.agent.grafana.com/.path: /path/to - profiles.agent.grafana.com/service_name: pyroscope - profiles.agent.grafana.com/tenant: primary - profiles.agent.grafana.com/scheme: http - profiles.agent.grafana.com/port: 8080 - profiles.agent.grafana.com/interval: 30s - profiles.agent.grafana.com/timeout: 15s -or - profiles.agent.grafana.com/.scrape: true - profiles.agent.grafana.com/.path: /path/to - profiles.agent.grafana.com/service_name: pyroscope - profiles.agent.grafana.com/tenant: primary - profiles.agent.grafana.com/scheme: http - profiles.agent.grafana.com/port: 8080 - profiles.agent.grafana.com/interval: 30s - profiles.agent.grafana.com/timeout: 15s - ``` -*/ -argument "forward_to" { - comment = "Must be a list(ProfilessReceiver) where collected profile should be forwarded to" -} - -argument "cluster" { - optional = true -} - -argument "namespace" { - optional = true -} - -argument "tenant" { - comment = "The tenant to write profile to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant label, and this can be a regex. (default: (.*))" - optional = true -} - -argument "scrape_interval" { - comment = "How often to scrape profile from the targets (default: 30s)" - optional = true -} - -argument "scrape_timeout" { - comment = "How long before a scrape times out (default: 15s)" - optional = true -} - -// get the available containers. -discovery.docker "dd_profiles" { - host = "unix:///var/run/docker.sock" - refresh_interval = "15s" - - filter { - name = "status" - values = ["running"] - } -} - -discovery.relabel "dr_docker_profiles" { - targets = discovery.docker.dd_profiles.targets - - // allow resources to declare the protocol to use when collecting profiles - // default value is "http" - rule { - action = "replace" - replacement = "http" - target_label = "__scheme__" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_scheme", - "__meta_docker_container_label_profiles_grafana_com_scheme", - ] - separator = ";" - regex = "^(?:;*)?(https?).*$" - replacement = "$1" - target_label = "__scheme__" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_port", - "__meta_docker_container_label_profiles_grafana_com_port", - ] - separator = ";" - regex = "^(?:;*)?(\\d+).*$" - replacement = "$1" - target_label = "__tmp_port" - } - - rule { - action = "replace" - source_labels = [ - "__address__", - "__tmp_port", - ] - separator = ";" - regex = "^([^:]+)(?::\\d+)?;(\\d+)$" - replacement = "$1:$2" - target_label = "__address__" - } - - // allow resources to declare their profiles should be sent to - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_tenant", - "__meta_docker_container_label_profiles_grafana_com_tenant", - ] - regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" - } - - // allow resources to declare how often their profiles should be collected. - // default value is 30s, the following duration formats are supported (s|m|ms|h|d) - rule { - action = "replace" - replacement = coalesce(argument.scrape_interval.value, "30s") - target_label = "__scrape_interval__" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_interval", - "__meta_docker_container_label_profiles_grafana_com_interval", - ] - separator = ";" - regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" - replacement = "$1" - target_label = "__scrape_interval__" - } - - // allow resources to declare the timeout of the scrape request - // default value is 15s, the following duration formats are supported (s|m|ms|h|d) - rule { - action = "replace" - replacement = coalesce(argument.scrape_timeout.value, "15s") - target_label = "__scrape_timeout__" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_interval", - "__meta_docker_container_label_profiles_grafana_com_interval", - ] - separator = ";" - regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" - replacement = "$1" - target_label = "__scrape_timeout__" - } - - /******************************************** - * Handle Setting Common Labels - ********************************************/ - - // set the cluster label - rule { - action = "replace" - replacement = coalesce(argument.cluster.value, "docker-compose") - target_label = "cluster" - } - - // set the namespace label - rule { - action = "replace" - replacement = coalesce(argument.namespace.value, "monitoring-system") - target_label = "namespace" - } - - // set a default job label to be the namespace/docker_compose_service - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = coalesce(argument.namespace.value, "monitoring-system") + "/$1" - target_label = "job" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "pod" - } - - rule { - source_labels = ["__meta_docker_container_name"] - regex = "/(.*)" - target_label = "container" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "app" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_app", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "app" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_com_docker_compose_service", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "service_name" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_service_name", - "__meta_docker_container_label_profiles_grafana_com_service_name", - "__meta_docker_container_label_pyroscope_io_service_name", - ] - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "service_name" - } -} - -/******************************************** - * Pyroscope Scrape CPU - ********************************************/ -discovery.relabel "dr_keep_cpu_targets" { - targets = discovery.relabel.dr_docker_profiles.output - - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_cpu_scrape", - "__meta_docker_container_label_profiles_grafana_com_cpu_scrape", - ] - regex = "^(?:;*)?(true).*$" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_cpu_path", - "__meta_docker_container_label_profiles_grafana_com_cpu_path", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "__profile_path__" - } -} - -pyroscope.scrape "ps_profile_cpu" { - forward_to = argument.forward_to.value - - job_name = "label-profiles-cpu" - targets = discovery.relabel.dr_keep_cpu_targets.output - - clustering { - enabled = true - } - - profiling_config { - profile.process_cpu { - enabled = true - } - - profile.memory { - enabled = false - } - - profile.goroutine { - enabled = false - } - - profile.block { - enabled = false - } - - profile.mutex { - enabled = false - } - - profile.fgprof { - enabled = false - } - } -} - -/******************************************** - * Pyroscope Scrape Memory - ********************************************/ -discovery.relabel "dr_keep_memory_targets" { - targets = discovery.relabel.dr_docker_profiles.output - - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_memory_scrape", - "__meta_docker_container_label_profiles_grafana_com_memory_scrape", - ] - regex = "^(?:;*)?(true).*$" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_memory_path", - "__meta_docker_container_label_profiles_grafana_com_memory_path", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "__profile_path__" - } -} - -pyroscope.scrape "ps_profile_memory" { - forward_to = argument.forward_to.value - - job_name = "label-profiles-memory" - targets = discovery.relabel.dr_keep_memory_targets.output - - clustering { - enabled = true - } - - profiling_config { - profile.process_cpu { - enabled = false - } - - profile.memory { - enabled = true - } - - profile.goroutine { - enabled = false - } - - profile.block { - enabled = false - } - - profile.mutex { - enabled = false - } - - profile.fgprof { - enabled = false - } - } -} - -/******************************************** - * Pyroscope Scrape Goroutine - ********************************************/ -discovery.relabel "dr_keep_goroutine_targets" { - targets = discovery.relabel.dr_docker_profiles.output - - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_goroutine_scrape", - "__meta_docker_container_label_profiles_grafana_com_goroutine_scrape", - ] - regex = "^(?:;*)?(true).*$" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_goroutine_path", - "__meta_docker_container_label_profiles_grafana_com_goroutine_path", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "__profile_path__" - } -} - -pyroscope.scrape "ps_profile_goroutine" { - forward_to = argument.forward_to.value - - job_name = "label-profiles-goroutine" - targets = discovery.relabel.dr_keep_goroutine_targets.output - - clustering { - enabled = true - } - - profiling_config { - profile.process_cpu { - enabled = false - } - - profile.memory { - enabled = false - } - - profile.goroutine { - enabled = true - } - - profile.block { - enabled = false - } - - profile.mutex { - enabled = false - } - - profile.fgprof { - enabled = false - } - } -} - -/******************************************** - * Pyroscope Scrape Block - ********************************************/ -discovery.relabel "dr_keep_block_targets" { - targets = discovery.relabel.dr_docker_profiles.output - - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_block_scrape", - "__meta_docker_container_label_profiles_grafana_com_block_scrape", - ] - regex = "^(?:;*)?(true).*$" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_block_path", - "__meta_docker_container_label_profiles_grafana_com_block_path", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "__profile_path__" - } -} - -pyroscope.scrape "ps_profile_block" { - forward_to = argument.forward_to.value - - job_name = "label-profiles-block" - targets = discovery.relabel.dr_keep_block_targets.output - - clustering { - enabled = true - } - - profiling_config { - profile.process_cpu { - enabled = false - } - - profile.memory { - enabled = false - } - - profile.goroutine { - enabled = false - } - - profile.block { - enabled = true - } - - profile.mutex { - enabled = false - } - - profile.fgprof { - enabled = false - } - } -} - -/******************************************** - * Pyroscope Scrape Mutex - ********************************************/ -discovery.relabel "dr_keep_mutex_targets" { - targets = discovery.relabel.dr_docker_profiles.output - - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_mutex_scrape", - "__meta_docker_container_label_profiles_grafana_com_mutex_scrape", - ] - regex = "^(?:;*)?(true).*$" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_mutex_path", - "__meta_docker_container_label_profiles_grafana_com_mutex_path", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "__profile_path__" - } -} - -pyroscope.scrape "ps_profile_mutex" { - forward_to = argument.forward_to.value - - job_name = "label-profiles-mutex" - targets = discovery.relabel.dr_keep_mutex_targets.output - - clustering { - enabled = true - } - - profiling_config { - profile.process_cpu { - enabled = false - } - - profile.memory { - enabled = false - } - - profile.goroutine { - enabled = false - } - - profile.block { - enabled = false - } - - profile.mutex { - enabled = true - } - - profile.fgprof { - enabled = false - } - } -} - -/******************************************** - * Pyroscope Scrape Fgprof - ********************************************/ -discovery.relabel "dr_keep_fgprof_targets" { - targets = discovery.relabel.dr_docker_profiles.output - - rule { - action = "keep" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_fgprof_scrape", - "__meta_docker_container_label_profiles_grafana_com_fgprof_scrape", - ] - regex = "^(?:;*)?(true).*$" - } - - rule { - action = "replace" - source_labels = [ - "__meta_docker_container_label_profiles_agent_grafana_com_fgprof_path", - "__meta_docker_container_label_profiles_grafana_com_fgprof_path", - ] - separator = ";" - regex = "^(?:;*)?([^;]+).*$" - replacement = "$1" - target_label = "__profile_path__" - } -} - -pyroscope.scrape "ps_profile_fgprof" { - forward_to = argument.forward_to.value - - job_name = "label-profiles-fgprof" - targets = discovery.relabel.dr_keep_fgprof_targets.output - - clustering { - enabled = true - } - - profiling_config { - profile.process_cpu { - enabled = false - } - - profile.memory { - enabled = false - } - - profile.goroutine { - enabled = false - } - - profile.block { - enabled = false - } - - profile.mutex { - enabled = false - } - - profile.fgprof { - enabled = true - } - } -} diff --git a/docker-compose/common/config/agent-flow/modules/grafana_cloud.river b/docker-compose/common/config/agent-flow/modules/grafana_cloud.river deleted file mode 100644 index 635d8829..00000000 --- a/docker-compose/common/config/agent-flow/modules/grafana_cloud.river +++ /dev/null @@ -1,114 +0,0 @@ -/* -Module: Grafana Cloud -Description: Grafana LGTMP Stack Receiver Provider - -To create a token: - 1. Navigate to the Grafana Cloud Portal: https://grafana.com/profile/org - 2. Go to either the Access Policies or API Keys page, located in the Security section - 3. Create an Access Policy or API token with the correct permissions - -The token must have permissions to read stack information. The setup of these permissions depends on the type of token: - Access Policies need the stacks:read scope - API Keys need at least the the MetricsPublisher role -*/ - -/******************************************** - * ARGUMENTS - ********************************************/ -argument "stack_name" { - comment = "Name of your stack as shown in the account console" -} - -argument "token" { - comment = "Access policy token or API Key." -} - -/******************************************** - * EXPORTS - ********************************************/ - -export "metrics_receiver" { - value = prometheus.remote_write.default.receiver -} - -export "logs_receiver" { - value = loki.write.default.receiver -} - -export "traces_receiver" { - value = otelcol.exporter.otlp.default.input -} - -export "profiles_receiver" { - value = pyroscope.write.default.receiver -} - -export "stack_information" { - value = json_decode(remote.http.config_file.content) -} - -/******************************************** - * External information - ********************************************/ - -remote.http "config_file" { - url = "https://grafana.com/api/instances/" + argument.stack_name.value - - client { - bearer_token = argument.token.value - } - poll_frequency = "24h" -} - -/******************************************** - * Endpoints - ********************************************/ - -// Metrics -prometheus.remote_write "default" { - endpoint { - url = json_decode(remote.http.config_file.content)["hmInstancePromUrl"] + "/api/prom/push" - - basic_auth { - username = json_decode(remote.http.config_file.content)["hmInstancePromId"] - password = argument.token.value - } - } -} - -// Logs -loki.write "default" { - endpoint { - url = json_decode(remote.http.config_file.content)["hlInstanceUrl"] + "/loki/api/v1/push" - - basic_auth { - username = json_decode(remote.http.config_file.content)["hlInstanceId"] - password = argument.token.value - } - } -} - -// Traces -otelcol.auth.basic "default" { - username = json_decode(remote.http.config_file.content)["htInstanceId"] - password = argument.token.value -} - -otelcol.exporter.otlp "default" { - client { - endpoint = json_decode(remote.http.config_file.content)["htInstanceUrl"] + ":443" - auth = otelcol.auth.basic.default.handler - } -} - -// Profiles -pyroscope.write "default" { - endpoint { - url = json_decode(remote.http.config_file.content)["hpInstanceUrl"] - - basic_auth { - username = json_decode(remote.http.config_file.content)["hpInstanceId"] - password = argument.token.value - } - } -} diff --git a/docker-compose/common/config/agent-flow/modules/local.river b/docker-compose/common/config/agent-flow/modules/local.river deleted file mode 100644 index 4dbdcfec..00000000 --- a/docker-compose/common/config/agent-flow/modules/local.river +++ /dev/null @@ -1,95 +0,0 @@ -/* -Module: Local -Description: Grafana LGTMP Stack Receiver Provider -*/ - -/******************************************** - * ARGUMENTS - ********************************************/ -argument "tenant" { - optional = true - default = "anonymous" -} - -argument "metrics_endpoint" { - comment = "Where to send collected metrics." - optional = true - default = "http://mimir:8080" -} - -argument "logs_endpoint" { - comment = "Where to send collected logs." - optional = true - default = "http://loki:3100" -} - -argument "traces_endpoint" { - comment = "Where to send collected traces." - optional = true - default = "tempo:4317" -} - -argument "profiles_endpoint" { - comment = "Where to send collected profiles." - optional = true - default = "http://pyroscope:4040" -} - -/******************************************** - * EXPORTS - ********************************************/ - -export "metrics_receiver" { - value = prometheus.remote_write.local.receiver -} - -export "logs_receiver" { - value = loki.write.local.receiver -} - -export "traces_receiver" { - value = otelcol.exporter.otlp.local.input -} - -export "profiles_receiver" { - value = pyroscope.write.local.receiver -} - -/******************************************** - * Endpoints - ********************************************/ - -// Metrics -prometheus.remote_write "local" { - endpoint { - url = argument.metrics_endpoint.value + "/api/v1/push" - send_native_histograms = true - } -} - -// Logs -loki.write "local" { - endpoint { - url = argument.logs_endpoint.value + "/loki/api/v1/push" - tenant_id = argument.tenant.value - } -} - -// Traces -otelcol.exporter.otlp "local" { - client { - endpoint = argument.traces_endpoint.value - - tls { - insecure = true - insecure_skip_verify = true - } - } -} - -// Profiles -pyroscope.write "local" { - endpoint { - url = argument.profiles_endpoint.value - } -} diff --git a/docker-compose/common/config/agent-flow/modules/provider.river b/docker-compose/common/config/agent-flow/modules/provider.river new file mode 100644 index 00000000..52fd8c59 --- /dev/null +++ b/docker-compose/common/config/agent-flow/modules/provider.river @@ -0,0 +1,194 @@ +/* +Module: Receiver Provider +Description: Grafana LGTMP Stack Receiver Provider +*/ + +declare "lgtmp_stack" { + + /******************************************** + * ARGUMENTS + ********************************************/ + argument "metrics_endpoint" { + comment = "Where to send collected metrics." + optional = true + default = "http://mimir:8080" + } + + argument "logs_endpoint" { + comment = "Where to send collected logs." + optional = true + default = "http://loki:3100" + } + + argument "traces_endpoint" { + comment = "Where to send collected traces." + optional = true + default = "tempo:4317" + } + + argument "profiles_endpoint" { + comment = "Where to send collected profiles." + optional = true + default = "http://pyroscope:4040" + } + + /******************************************** + * Setup Receivers + ********************************************/ + prometheus.remote_write "lgtmp_stack" { + endpoint { + url = argument.metrics_endpoint.value + "/api/v1/push" + send_native_histograms = true + } + } + + loki.write "lgtmp_stack" { + endpoint { + url = argument.logs_endpoint.value + "/loki/api/v1/push" + } + } + + otelcol.exporter.otlp "lgtmp_stack" { + client { + endpoint = argument.traces_endpoint.value + + tls { + insecure = true + insecure_skip_verify = true + } + } + } + + pyroscope.write "lgtmp_stack" { + endpoint { + url = argument.profiles_endpoint.value + } + } + + /******************************************** + * EXPORTS + ********************************************/ + export "metrics_receiver" { + value = prometheus.remote_write.lgtmp_stack.receiver + } + + export "logs_receiver" { + value = loki.write.lgtmp_stack.receiver + } + + export "traces_receiver" { + value = otelcol.exporter.otlp.lgtmp_stack.input + } + + export "profiles_receiver" { + value = pyroscope.write.lgtmp_stack.receiver + } +} + +declare "grafana_cloud" { + + /* + To create a token: + 1. Navigate to the Grafana Cloud Portal: https://grafana.com/profile/org + 2. Go to either the Access Policies or API Keys page, located in the Security section + 3. Create an Access Policy or API token with the correct permissions + + The token must have permissions to read stack information. The setup of these permissions depends on the type of token: + Access Policies need the stacks:read scope + API Keys need at least the the MetricsPublisher role + */ + + /******************************************** + * ARGUMENTS + ********************************************/ + argument "stack_name" { + comment = "Name of your stack as shown in the account console" + } + + argument "token" { + comment = "Access policy token or API Key." + } + + /******************************************** + * External information + ********************************************/ + remote.http "config_file" { + url = "https://grafana.com/api/instances/" + argument.stack_name.value + + client { + bearer_token = argument.token.value + } + poll_frequency = "24h" + } + + /******************************************** + * Setup Receivers + ********************************************/ + prometheus.remote_write "grafana_cloud" { + endpoint { + url = json_decode(remote.http.config_file.content)["hmInstancePromUrl"] + "/api/prom/push" + + basic_auth { + username = json_decode(remote.http.config_file.content)["hmInstancePromId"] + password = argument.token.value + } + } + } + + loki.write "grafana_cloud" { + endpoint { + url = json_decode(remote.http.config_file.content)["hlInstanceUrl"] + "/loki/api/v1/push" + + basic_auth { + username = json_decode(remote.http.config_file.content)["hlInstanceId"] + password = argument.token.value + } + } + } + + otelcol.auth.basic "grafana_cloud" { + username = json_decode(remote.http.config_file.content)["htInstanceId"] + password = argument.token.value + } + + otelcol.exporter.otlp "grafana_cloud" { + client { + endpoint = json_decode(remote.http.config_file.content)["htInstanceUrl"] + ":443" + auth = otelcol.auth.basic.grafana_cloud.handler + } + } + + pyroscope.write "grafana_cloud" { + endpoint { + url = json_decode(remote.http.config_file.content)["hpInstanceUrl"] + + basic_auth { + username = json_decode(remote.http.config_file.content)["hpInstanceId"] + password = argument.token.value + } + } + } + + /******************************************** + * EXPORTS + ********************************************/ + export "metrics_receiver" { + value = prometheus.remote_write.grafana_cloud.receiver + } + + export "logs_receiver" { + value = loki.write.grafana_cloud.receiver + } + + export "traces_receiver" { + value = otelcol.exporter.otlp.grafana_cloud.input + } + + export "profiles_receiver" { + value = pyroscope.write.grafana_cloud.receiver + } + + export "stack_information" { + value = json_decode(remote.http.config_file.content) + } +} diff --git a/docker-compose/common/config/agent-flow/monolithic-mode-all.river b/docker-compose/common/config/agent-flow/monolithic-mode-all.river index ca23bb81..33693858 100644 --- a/docker-compose/common/config/agent-flow/monolithic-mode-all.river +++ b/docker-compose/common/config/agent-flow/monolithic-mode-all.river @@ -8,27 +8,16 @@ logging { /******************************************** * Grafana LGTMP Stack Receiver Provider ********************************************/ -module.file "lgtmp_provider_local" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/local.river" - - arguments { - logs_endpoint = "http://gateway:3100" - metrics_endpoint = "http://gateway:8080" - traces_endpoint = "gateway:4317" - profiles_endpoint = "http://gateway:4040" - } +import.file "provider" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/provider.river" } -/* -module.file "lgtmp_provider_grafana_cloud" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/grafana_cloud.river" - - arguments { - stack_name = "" - token = "" - } +provider.lgtmp_stack "docker_compose" { + logs_endpoint = "http://gateway:3100" + metrics_endpoint = "http://gateway:8080" + traces_endpoint = "gateway:4317" + profiles_endpoint = "http://gateway:4040" } -*/ /******************************************** * Metrics @@ -37,7 +26,7 @@ module.file "metrics_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river" arguments { - forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver] + forward_to = [provider.lgtmp_stack.docker_compose.metrics_receiver] } } @@ -48,10 +37,7 @@ module.file "logs_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/logs/all.river" arguments { - forward_to = [ - module.file.lgtmp_provider_local.exports.logs_receiver, - // module.file.lgtmp_provider_grafana_cloud.exports.logs_receiver, - ] + forward_to = [provider.lgtmp_stack.docker_compose.logs_receiver] } } @@ -62,9 +48,9 @@ module.file "traces_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/traces/all.river" arguments { - metrics_forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver] - logs_forward_to = [module.file.lgtmp_provider_local.exports.logs_receiver] - traces_forward_to = [module.file.lgtmp_provider_local.exports.traces_receiver] + metrics_forward_to = [provider.lgtmp_stack.docker_compose.metrics_receiver] + logs_forward_to = [provider.lgtmp_stack.docker_compose.logs_receiver] + traces_forward_to = [provider.lgtmp_stack.docker_compose.traces_receiver] } } @@ -80,6 +66,6 @@ module.file "profiles_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/profiles/all.river" arguments { - forward_to = [module.file.lgtmp_provider_local.exports.profiles_receiver] + forward_to = [provider.lgtmp_stack.docker_compose.profiles_receiver] } } diff --git a/docker-compose/common/config/agent-flow/profiles.river b/docker-compose/common/config/agent-flow/profiles.river index dc34590b..1070f2a2 100644 --- a/docker-compose/common/config/agent-flow/profiles.river +++ b/docker-compose/common/config/agent-flow/profiles.river @@ -8,12 +8,13 @@ logging { /******************************************** * Grafana LGTMP Stack Receiver Provider ********************************************/ -module.file "lgtmp_provider_local" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/local.river" +import.file "provider" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/provider.river" +} - arguments { - profiles_endpoint = "http://gateway:4040" - } +provider.lgtmp_stack "docker_compose" { + metrics_endpoint = "http://gateway:8080" + profiles_endpoint = "http://gateway:4040" } /******************************************** @@ -23,7 +24,7 @@ module.file "profiles_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/profiles/all.river" arguments { - forward_to = [module.file.lgtmp_provider_local.exports.profiles_receiver] + forward_to = [provider.lgtmp_stack.docker_compose.profiles_receiver] } } @@ -34,6 +35,6 @@ module.file "metrics_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river" arguments { - forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver] + forward_to = [provider.lgtmp_stack.docker_compose.metrics_receiver] } } diff --git a/docker-compose/common/config/agent-flow/traces.river b/docker-compose/common/config/agent-flow/traces.river index 4eaa585f..e5f1287c 100644 --- a/docker-compose/common/config/agent-flow/traces.river +++ b/docker-compose/common/config/agent-flow/traces.river @@ -8,13 +8,14 @@ logging { /******************************************** * Grafana LGTMP Stack Receiver Provider ********************************************/ -module.file "lgtmp_provider_local" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/local.river" +import.file "provider" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/provider.river" +} - arguments { - metrics_endpoint = "http://gateway:8080" - traces_endpoint = "gateway:4317" - } +provider.lgtmp_stack "docker_compose" { + logs_endpoint = "http://gateway:3100" + metrics_endpoint = "http://gateway:8080" + traces_endpoint = "gateway:4317" } /******************************************** @@ -24,9 +25,9 @@ module.file "traces_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/traces/all.river" arguments { - metrics_forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver] - logs_forward_to = [module.file.lgtmp_provider_local.exports.logs_receiver] - traces_forward_to = [module.file.lgtmp_provider_local.exports.traces_receiver] + metrics_forward_to = [provider.lgtmp_stack.docker_compose.metrics_receiver] + logs_forward_to = [provider.lgtmp_stack.docker_compose.logs_receiver] + traces_forward_to = [provider.lgtmp_stack.docker_compose.traces_receiver] } } @@ -37,6 +38,6 @@ module.file "metrics_primary" { filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river" arguments { - forward_to = [module.file.lgtmp_provider_local.exports.metrics_receiver] + forward_to = [provider.lgtmp_stack.docker_compose.metrics_receiver] } }