feat: detect json and logfmt log types

modules/kubernetes/annotations/logs/README.md

@@ -427,10 +427,37 @@ log_annotations.decolorize "default" {
   annotation = "logs.grafana.com"
 }
 
+// set default level and log_type to unknown
 log_utils.default_level "default" {
+  forward_to = [log_utils.klog_format.default.receiver]
+}
+
+// identify klog format and parse log level
+log_utils.klog_format "default" {
+  forward_to = [log_utils.zerolog_format.default.receiver]
+}
+
+// identify zerolog format and parse log level
+log_utils.zerolog_format "default" {
+  forward_to = [log_utils.json_format.default.receiver]
+}
+
+// identify json format and parse log level
+log_utils.json_format "default" {
+  forward_to = [log_utils.logfmt_format.default.receiver]
+}
+
+// identify logfmt format and parse log level
+log_utils.logfmt_format "default" {
+  forward_to = [log_utils.unknown_format.default.receiver]
+}
+
+// attempt parse log level from unidentified log type
+log_utils.unknown_format "default" {
   forward_to = [log_utils.normalize_level.default.receiver]
 }
 
+// Set level consistently (case and name)
 log_utils.normalize_level "default" {
   forward_to = [
     log_utils.pre_process_metrics.default.receiver,

modules/utils/logs/log-levels.alloy

@@ -17,21 +17,41 @@ declare "default_level" {
     forward_to = argument.forward_to.value
 
     /*******************************************************************************
-    *                         Log-Level Parsing
+    *                       Set baseline level and log_type
     ********************************************************************************/
-    // default level to unknown
+    // default level to value or default_level (default: unknown)
     stage.static_labels {
       values = {
         level = argument.default_level.value,
       }
     }
 
-    // default level to unknown
+    // default type to unknown
     stage.static_labels {
       values = {
         log_type = "unknown",
       }
     }
+  }
+}
+
+declare "klog_format" {
+  argument "forward_to" {
+    comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+  }
+
+  argument "default_level" {
+    comment = "The default log level to use if one is not set (default: unknown)"
+    optional = true
+    default = "unknown"
+  }
+
+  export "receiver" {
+    value = loki.process.format_klog.receiver
+  }
+
+  loki.process "format_klog" {
+    forward_to = argument.forward_to.value
 
     // check to see if the log line matches the klog format (https://github.com/kubernetes/klog)
     stage.match {
@@ -85,6 +105,26 @@ declare "default_level" {
         }
       }
     }
+  }
+}
+
+declare "zerolog_format" {
+  argument "forward_to" {
+    comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+  }
+
+  argument "default_level" {
+    comment = "The default log level to use if one is not set (default: unknown)"
+    optional = true
+    default = "unknown"
+  }
+
+  export "receiver" {
+    value = loki.process.format_zerolog.receiver
+  }
+
+  loki.process "format_zerolog" {
+    forward_to = argument.forward_to.value
 
     // check to see if the log line matches the zerolog format
     stage.match {
@@ -111,6 +151,119 @@ declare "default_level" {
         }
       }
     }
+  }
+}
+
+declare "json_format" {
+  argument "forward_to" {
+    comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+  }
+
+  argument "default_level" {
+    comment = "The default log level to use if one is not set (default: unknown)"
+    optional = true
+    default = "unknown"
+  }
+
+  export "receiver" {
+    value = loki.process.format_json.receiver
+  }
+
+  loki.process "format_json" {
+    forward_to = argument.forward_to.value
+
+    // check to see if the log line matches the json format
+    stage.match {
+      // unescaped regex: ^\s*\{.+\}\s*$
+      selector = "{level=\"" + argument.default_level.value + "\"} |~ \"^\\\\s*\\\\{.+\\\\}\\\\s*$\""
+
+      // set the log_type
+      stage.static_labels{
+        values = {
+          log_type = "json",
+        }
+      }
+
+      // extract the level
+      stage.json {
+        expressions = {
+          level = "level || lvl || loglevel || LogLevel || log_level || logLevel || log_lvl || logLvl || levelname || levelName || LevelName",
+        }
+      }
+
+      // set the extracted level as a label
+      stage.labels {
+        values = {
+          level = "",
+        }
+      }
+    }
+  }
+}
+
+declare "logfmt_format" {
+  argument "forward_to" {
+    comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+  }
+
+  argument "default_level" {
+    comment = "The default log level to use if one is not set (default: unknown)"
+    optional = true
+    default = "unknown"
+  }
+
+  export "receiver" {
+    value = loki.process.format_logfmt.receiver
+  }
+
+  loki.process "format_logfmt" {
+    forward_to = argument.forward_to.value
+
+    // check to see if the log line matches the logfmt format
+    stage.match {
+      // unescaped regex: ^(\w+=("[^"]*"|\S+))(\s+(\w+=("[^"]*"|\S+)))*\s*
+      selector = "{level=\"" + argument.default_level.value + "\"} |~ \"^(\\\\w+=(\\\"[^\\\"]*\\\"|\\\\S+))(\\\\s+(\\\\w+=(\\\"[^\\\"]*\\\"|\\\\S+)))*\\\\s*\""
+
+      // set the log_type
+      stage.static_labels{
+        values = {
+          log_type = "logfmt",
+        }
+      }
+
+      // while the level could be extracted as logfmt, this allows for multiple possible log levels formats
+      // i.e. loglevel=info, level=info, lvl=info, loglvl=info
+      stage.regex {
+        expression = "(log)?(level|lvl)=\"?(?P<level>\\S+)\"?"
+      }
+
+      // set the extracted level value as a label
+      stage.labels {
+        values = {
+          level = "",
+        }
+      }
+    }
+  }
+}
+
+declare "unknown_format" {
+  argument "forward_to" {
+    comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to"
+  }
+
+  argument "default_level" {
+    comment = "The default log level to use if one is not set (default: unknown)"
+    optional = true
+    default = "unknown"
+  }
+
+  export "receiver" {
+    value = loki.process.format_unknown.receiver
+  }
+
+  loki.process "format_unknown" {
+    forward_to = argument.forward_to.value
 
     // if the level is still unknown, do one last attempt at detecting it based on common levels
     stage.match {