Merge pull request #47 from qclaogui:log-ingestion-via-labels

Docker Compose(logs): Log Ingestion via Labels

docker-compose/common/compose-include/agent-collect-logs.yaml

@@ -1,6 +1,8 @@
 
 services:
   agent:
+    labels:
+      - logs.agent.grafana.com/log-format=logfmt
     depends_on:
       gateway:
         condition: service_healthy

docker-compose/common/compose-include/minio.yaml

@@ -1,6 +1,8 @@
 
 services:
   minio:
+    labels:
+      - logs.agent.grafana.com/scrape=false
     image: minio/minio:RELEASE.2024-03-05T04-48-44Z
     entrypoint:
       - sh

docker-compose/common/config/agent-flow/logs.river

@@ -1,10 +1,14 @@
 // https://github.com/grafana/agent-configurator
 
 logging {
-	level  = "warn"
+	level  = "info"
 	format = "logfmt"
 }
 
+/********************************************
+ * LGTMP Receiver provider
+ ********************************************/
+
 module.file "docker_compose" {
 	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker_compose.river"
 
@@ -13,72 +17,14 @@ module.file "docker_compose" {
 	}
 }
 
-discovery.relabel "containers" {
-	targets = module.file.docker_compose.exports.relabelings_common.output
-}
-
 /********************************************
  * Logs
  ********************************************/
 
-loki.source.docker "containers" {
-	host          = "unix:///var/run/docker.sock"
-	targets       = discovery.relabel.containers.output
-	relabel_rules = discovery.relabel.containers.rules
-	forward_to    = [loki.process.containers.receiver]
-}
-
-loki.process "containers" {
-	forward_to = [module.file.docker_compose.exports.logs_receiver]
-
-	stage.drop {
-		longer_than = "8KB"
-		older_than  = "12h"
-	}
-
-	stage.tenant {
-		value = "anonymous"
-	}
-}
-
-/********************************************
- * Otelcol for Logs
- ********************************************/
-
-otelcol.receiver.otlp "containers" {
-	grpc {
-		endpoint = "0.0.0.0:4317"
-	}
-
-	http {
-		endpoint = "0.0.0.0:4318"
-	}
-
-	output {
-		logs = [otelcol.processor.batch.containers.input]
-	}
-}
-
-otelcol.processor.batch "containers" {
-	send_batch_size     = 16384
-	send_batch_max_size = 0
-	timeout             = "2s"
+module.file "logs_primary" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/all.river"
 
-	output {
-		logs = [otelcol.processor.memory_limiter.containers.input]
-	}
-}
-
-otelcol.processor.memory_limiter "containers" {
-	check_interval         = "1s"
-	limit_percentage       = 50
-	spike_limit_percentage = 30
-
-	output {
-		logs = [otelcol.exporter.loki.containers.input]
+	arguments {
+		forward_to = [module.file.docker_compose.exports.logs_receiver]
 	}
 }
-
-otelcol.exporter.loki "containers" {
-	forward_to = [loki.process.containers.receiver]
-}

docker-compose/common/config/agent-flow/modules/docker/README.md

@@ -0,0 +1,24 @@
+# Docker Compose Modules
+
+## Logs
+
+The following service labels are supported:
+
+| Label            | Description |
+| :--------------- | :-----------|
+| `logs.agent.grafana.com/scrape` | Allow a service to declare it's logs should be dropped. |
+| `logs.agent.grafana.com/tenant` | Allow a service to override the tenant for its logs. |
+| `logs.agent.grafana.com/log-format` | If specified additional processing is performed to extract details based on the specified format.  This value can be a comma-delimited list, in the instances a pod may have multiple containers.  The following formats are currently supported: <ul><li>common-log<li>donet<li>istio<li>json<li>klog<li>log4j-json<li>logfmt<li>otel<li>postgres<li>python<li>spring-boot<li>syslog<li>zerolog</ul> |
+| `logs.agent.grafana.com/scrub-level` | Boolean whether or not the level should be dropped from the log message (as it is a label). |
+| `logs.agent.grafana.com/scrub-timestamp` | Boolean whether or not the timestamp should be dropped from the log message (as it is metadata). |
+| `logs.agent.grafana.com/scrub-nulls` | Boolean whether or not keys with null values should be dropped from json, reducing the size of the log message. |
+| `logs.agent.grafana.com/scrub-empties` | Boolean whether or not keys with empty values (`"", [], {}`) should be dropped from json, reducing the size of the log message. |
+| `logs.agent.grafana.com/drop-info` | Boolean whether or not info messages should be dropped (default is `false`), but a pod can override this temporarily or permanently. |
+| `logs.agent.grafana.com/drop-debug` | Boolean whether or not debug messages should be dropped (default is `true`), but a pod can override this temporarily or permanently. |
+| `logs.agent.grafana.com/drop-trace` | Boolean whether or not trace messages should be dropped (default is `true`), but a pod can override this temporarily or permanently. |
+| `logs.agent.grafana.com/mask-ssn` | Boolean whether or not to mask SSNs in the log line, if true the data will  be masked as `*SSN*salt*` |
+| `logs.agent.grafana.com/mask-credit-card` | Boolean whether or not to mask credit cards in the log line, if true the data will be masked as `*credit-card*salt*` |
+| `logs.agent.grafana.com/mask-email` | Boolean whether or not to mask emails in the log line, if true the data will be masked as`*email*salt*` |
+| `logs.agent.grafana.com/mask-ipv4` | Boolean whether or not to mask IPv4 addresses in the log line, if true the data will be masked as`*ipv4*salt*` |
+| `logs.agent.grafana.com/mask-ipv6` | Boolean whether or not to mask IPv6 addresses in the log line, if true the data will be masked as `*ipv6*salt*` |
+| `logs.agent.grafana.com/mask-phone` | Boolean whether or not to mask phone numbers in the log line, if true the data will be masked as `*phone*salt*` |

docker-compose/common/config/agent-flow/modules/docker/logs/all.river

@@ -0,0 +1,118 @@
+/*
+Module: log-all
+Description: Wrapper module to include all docker logging modules and use cri parsing
+*/
+argument "forward_to" {
+	// comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+	optional = false
+}
+
+argument "tenant" {
+	// comment = "The tenant to filter logs to.  This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex."
+	optional = true
+	default  = ".*"
+}
+
+argument "keep_labels" {
+	// comment = "List of labels to keep before the log message is written to Loki"
+	optional = true
+	default  = [
+		"app",
+		"cluster",
+		"component",
+		"container",
+		"deployment",
+		"env",
+		"filename",
+		"instance",
+		"job",
+		"level",
+		"log_type",
+		"namespace",
+		"region",
+		"service",
+		"squad",
+		"team",
+	]
+}
+
+module.file "log_targets" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/targets/logs-from-docker.river"
+
+	arguments {
+		forward_to = [module.file.log_formats_all.exports.process.receiver]
+		tenant     = argument.tenant.value
+	}
+}
+
+module.file "log_formats_all" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/log-formats/all.river"
+
+	arguments {
+		forward_to = [module.file.log_level_default.exports.process.receiver]
+	}
+}
+
+module.file "log_level_default" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/labels/log-level.river"
+
+	arguments {
+		// here we fork, one branch goes to the log level module, the other goes to the metrics module
+		// this is because we need to reduce the labels on the pre-metrics but they are still necessary in
+		// downstream modules
+		forward_to = [
+			module.file.pre_process_metrics.exports.process.receiver,
+			module.file.drop_levels.exports.process.receiver,
+		]
+	}
+}
+
+module.file "pre_process_metrics" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/metrics/pre-process-bytes-lines.river"
+
+	arguments {
+		forward_to  = [module.file.drop_levels.exports.process.receiver]
+		keep_labels = argument.keep_labels.value
+	}
+}
+
+module.file "drop_levels" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/drops/levels.river"
+
+	arguments {
+		forward_to = [module.file.scrub_all.exports.process.receiver]
+	}
+}
+
+module.file "scrub_all" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/scrubs/all.river"
+
+	arguments {
+		forward_to = [module.file.mask_all.exports.process.receiver]
+	}
+}
+
+module.file "mask_all" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/masks/all.river"
+
+	arguments {
+		forward_to = [module.file.label_keep.exports.process.receiver]
+	}
+}
+
+module.file "label_keep" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/labels/keep-labels.river"
+
+	arguments {
+		forward_to  = [module.file.post_process_metrics.exports.process.receiver]
+		keep_labels = argument.keep_labels.value
+	}
+}
+
+module.file "post_process_metrics" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/metrics/post-process-bytes-lines.river"
+
+	arguments {
+		forward_to = argument.forward_to.value
+	}
+}

docker-compose/common/config/agent-flow/modules/docker/logs/drops/level-debug.river

@@ -0,0 +1,28 @@
+/*
+Module: drop-debug
+Description: The default behavior is to drop debug level messaging automatically, however, debug level
+             messages can still be logged by adding the annotation:
+
+             logs.agent.grafana.com/drop-debug: false
+*/
+argument "forward_to" {
+	// comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+	optional = false
+}
+
+export "process" {
+	value = loki.process.drop_debug
+}
+
+loki.process "drop_debug" {
+	forward_to = argument.forward_to.value
+
+	// check logs.agent.grafana.com/drop-debug annotation, if not set or set to true then drop
+	// any log message with level=debug
+	stage.match {
+		pipeline_name       = "pipeline for annotation || logs.agent.grafana.com/drop-debug: true"
+		selector            = "{level=~\"(?i)debug?\",logs_agent_grafana_com_drop_debug!=\"false\"}"
+		action              = "drop"
+		drop_counter_reason = "debug"
+	}
+}

docker-compose/common/config/agent-flow/modules/docker/logs/drops/level-info.river

@@ -0,0 +1,28 @@
+/*
+Module: drop-info
+Description: The default behavior is to keep info level messaging automatically, however, info level
+             messages can dropped by adding the annotation:
+
+             logs.agent.grafana.com/drop-info: true
+*/
+argument "forward_to" {
+	// comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+	optional = false
+}
+
+export "process" {
+	value = loki.process.drop_info
+}
+
+loki.process "drop_info" {
+	forward_to = argument.forward_to.value
+
+	// check logs.agent.grafana.com/drop-info annotation, if not set or set to true then drop
+	// any log message with level=info
+	stage.match {
+		pipeline_name       = "pipeline for annotation || logs.agent.grafana.com/drop-info: true"
+		selector            = "{level=~\"(?i)info?\",logs_agent_grafana_com_drop_info=\"true\"}"
+		action              = "drop"
+		drop_counter_reason = "info"
+	}
+}

docker-compose/common/config/agent-flow/modules/docker/logs/drops/level-trace.river

@@ -0,0 +1,28 @@
+/*
+Module: drop-trace
+Description: The default behavior is to drop trace level messaging automatically, however, trace level
+             messages can still be logged by adding the annotation:
+
+             logs.agent.grafana.com/drop-trace: false
+*/
+argument "forward_to" {
+	// comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+	optional = false
+}
+
+export "process" {
+	value = loki.process.drop_trace
+}
+
+loki.process "drop_trace" {
+	forward_to = argument.forward_to.value
+
+	// check logs.agent.grafana.com/drop-trace annotation, if not set or set to true then drop
+	// any log message with level=trace
+	stage.match {
+		pipeline_name       = "pipeline for annotation || logs.agent.grafana.com/drop-trace: true"
+		selector            = "{level=~\"(?i)trace?\",logs_agent_grafana_com_drop_trace!=\"false\"}"
+		action              = "drop"
+		drop_counter_reason = "trace"
+	}
+}

docker-compose/common/config/agent-flow/modules/docker/logs/drops/levels.river

@@ -0,0 +1,36 @@
+/*
+Module: drop-levels
+Description: Wrapper module to include all drop level modules
+*/
+argument "forward_to" {
+	// comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+	optional = false
+}
+
+export "process" {
+	value = module.file.drop_trace.exports.process
+}
+
+module.file "drop_trace" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/drops/level-trace.river"
+
+	arguments {
+		forward_to = [module.file.drop_debug.exports.process.receiver]
+	}
+}
+
+module.file "drop_debug" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/drops/level-debug.river"
+
+	arguments {
+		forward_to = [module.file.drop_info.exports.process.receiver]
+	}
+}
+
+module.file "drop_info" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker/logs/drops/level-info.river"
+
+	arguments {
+		forward_to = argument.forward_to.value
+	}
+}