kubernetes_container.yaml

version: 0.0.17
title: Kubernetes Container
description: Log parser for Kubernetes
min_stanza_version: 0.9.8
supported_platforms:
  - kubernetes
parameters:
  - name: cluster_name
    label: Cluster Name
    description: Cluster Name to be added to a resource label
    type: string
  - name: pod_name
    label: Pod Name
    description: Specify a single name or use * to collect logs from all pods.
    type: string
    default: "*"
  - name: container_name
    label: Container Name
    description: Specify a single name or use * to collect logs from all containers.
    type: string
    default: "*"
  - name: exclude
    label: Exclude Files
    description: A pattern of files to exclude
    type: strings
    required: false
  - name: enable_nested_json_parser
    label: Enable JSON Parsing
    description: Enable to this to attempt to parse JSON from log message.
    type: bool
    default: true
  - name: start_at
    label: Start At
    description: Start reading file from 'beginning' or 'end'
    type: enum
    valid_values:
      - beginning
      - end
    default: end

# Set Defaults
# {{ $cluster_name := default "" .cluster_name }}
# {{ $pod_name := default "*" .pod_name }}
# {{ $container_name := default "*" .container_name }}
# {{ $enable_nested_json_parser := default true .enable_nested_json_parser}}
# {{ $start_at := default "end" .start_at }}

# Pipeline Template
pipeline:
  - id: container_reader
    type: file_input
    include:
      - '/var/log/containers/{{ $pod_name }}-*_{{ $container_name }}-*.log'
    exclude:
      - '/var/log/containers/kube*'
    # {{ range $index, $element := .exclude }}
      - {{ $element }}
    # {{ end }}
    start_at: '{{ $start_at }}'
    labels:
      log_type: 'k8s.container'
      plugin_id: '{{ .id }}'
    resource:
      k8s.node.name: "EXPR(env('NODE_NAME'))"
      k8s.cluster.name: "{{ $cluster_name }}"

  - id: container_type_router
    type: router
    routes:
      # Route to containerd parser if it matches containerd log format
      - expr: '$record matches "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3,9}([\\+-]\\d{2}:\\d{2}|Z) std(?:out|err) [FP] .*"'
        output: containerd_parser
    default: docker_json_parser

  # Parse containerd log from $record
  - id: containerd_parser
    type: regex_parser
    regex: '^(?P<time>[^\s]+) (?P<stream>\w+) (?P<partial>\w) (?P<log>.*)'

  # Recombine multiline containerd log messages
  - id: containerd_recombine
    type: recombine
    combine_field: log
    is_last_entry: "$record.partial == 'F'"

  # Route Containerd time formats
  - id: time_parser_router
    type: router
    routes:
      - output: local_containerd_timestamp_parser
        expr: '$record.time != nil and $record.time matches "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3,9}[\\+-]\\d{2}:\\d{2}"'
      - output: utc_containerd_timestamp_parser
        expr: '$record.time != nil and $record.time matches "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3,9}Z"'

  # Containerd can have a couple timestamp formats depending if the node has local time set
  - id: local_containerd_timestamp_parser
    type: time_parser
    parse_from: time
    layout: '%Y-%m-%dT%H:%M:%S.%s%j'
    output: sev_parser

  # Containerd can have a couple timestamp formats depending if the node has local time set
  - id: utc_containerd_timestamp_parser
    type: time_parser
    parse_from: time
    layout: '%Y-%m-%dT%H:%M:%S.%sZ'
    output: sev_parser

  # Parse docker log format
  - id: docker_json_parser
    type: json_parser

  # Parse time from docker log format
  - id: docker_timestamp_parser
    type: time_parser
    parse_from: time
    layout: '%Y-%m-%dT%H:%M:%S.%sZ'

  # $record.stream should exist now so parse it.
  - id: sev_parser
    type: severity_parser
    parse_from: stream
    preserve_to: '$labels.stream'
    mapping:
      error: stderr
      info: stdout

  # Parse file name to grab resource information
  - id: file_name_parser
    type: regex_parser
    parse_from: $labels.file_name
    regex: '^(?P<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?P<namespace>[^_]+)_(?P<container_name>.+)-(?P<container_id>[a-z0-9]{64})\.log$'

  # Move resource information from $record to $resource
  - id: resource_restructurer
    type: restructure
    ops:
      - move:
          from: '$record.pod_name'
          to: '$resource["k8s.pod.name"]'
      - move:
          from: '$record.namespace'
          to: '$resource["k8s.namespace.name"]'
      - move:
          from: '$record.container_name'
          to: '$resource["k8s.container.name"]'
      - move:
          from: '$record.container_id'
          to: '$resource["k8s.container.id"]'

# {{ if $enable_nested_json_parser }}
  # Attempt to parse nested JSON in log field if it exists and if JSON is detected
  - id: log_json_router
    type: router
    default: move_log_to_record
    routes:
      # It appears to be JSON so send it to be parsed as JSON.
      - output: nested_json_parser
        expr: '$record.log != nil and $record.log matches "^{.*}\\s*$"'

  # Attempt to parse nested JSON since the log appears to be JSON
  - id: nested_json_parser
    type: json_parser
    parse_from: $record.log
    output: add_kubernetes_metadata
# {{ end }}

  # Move message from log field to $record
  - id: move_log_to_record
    type: restructure
    ops:
      - move:
          from: log
          to: '$record'

  # Add kubernetes metadata
  - id: add_kubernetes_metadata
    type: k8s_metadata_decorator
    output: {{ .output }}