Skip to content

Commit

Permalink
Merge pull request #1 from tuenti/yaml-syntax
Browse files Browse the repository at this point in the history 
Generic plugins syntax
  • Loading branch information
@Wiston999
Wiston999 authored Jun 20, 2023
2 parents 0fe6021 + d0d8265 commit d5f9f63
Show file tree
Hide file tree
Showing 36 changed files with 457 additions and 942 deletions.
107 changes: 60 additions & 47 deletions data/common.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,235 +1,248 @@
---

fluentbit::manage_package_repo: true
fluentbit::repo_key_fingerprint: F209D8762A60CD49E680633B4FF8368B6EA0722A
fluentbit::repo_key_fingerprint: C3C0A28534B9293EAF51FABD9F9DDC083888C1CD
fluentbit::repo_key_location: 'https://packages.fluentbit.io/fluentbit.key'
fluentbit::package_ensure: present
fluentbit::package_name: 'td-agent-bit'
fluentbit::package_name: 'fluent-bit'

fluentbit::manage_service: true
fluentbit::service_enable: true
fluentbit::service_has_status: true
fluentbit::service_has_restart: true
fluentbit::service_restart_command: 'systemctl reload-or-restart fluent-bit'
fluentbit::service_ensure: 'running'
fluentbit::service_name: 'td-agent-bit'
fluentbit::service_name: 'fluent-bit'
fluentbit::manage_config_dir: true
fluentbit::manage_data_dir: true
fluentbit::manage_storage_dir: true

fluentbit::config_file: '/etc/td-agent-bit/td-agent-bit.conf'
fluentbit::binary_file: '/opt/fluent-bit/bin/fluent-bit'
fluentbit::data_dir: '/opt/fluent-bit/db'
fluentbit::config_file: '/etc/fluent-bit/fluent-bit.conf'
fluentbit::config_file_mode: '0644'
fluentbit::config_folder_mode: '0750'
fluentbit::flush: 5
fluentbit::grace: 5
fluentbit::daemon: false
fluentbit::log_file: ~
fluentbit::dns_mode: UDP
fluentbit::log_level: info
fluentbit::http_server: false
fluentbit::http_listen: '0.0.0.0'
fluentbit::http_port: 2020
fluentbit::coro_stack_size: 24576
fluentbit::scheduler_cap: 2000
fluentbit::scheduler_base: 5
fluentbit::json_convert_nan_to_null: false

fluentbit::variables: {}

fluentbit::storage_path: ~
fluentbit::storage_sync: ~
fluentbit::storage_path: '/opt/fluent-bit/storage'
fluentbit::storage_sync: normal
fluentbit::storage_checksum: false
fluentbit::storage_backlog_mem_limit: ~
fluentbit::storage_max_chunks_up: 128
fluentbit::storage_backlog_mem_limit: 5M
fluentbit::storage_metrics: true
fluentbit::storage_delete_irrecoverable_chunks: false

fluentbit::input_plugins: []
fluentbit::output_plugins: []
fluentbit::filter_plugins: []
fluentbit::input_plugins: {}
fluentbit::output_plugins: {}
fluentbit::filter_plugins: {}

fluentbit::manage_plugins_file: true
fluentbit::plugins_file: '/etc/td-agent-bit/plugins.conf'
fluentbit::plugins_file: '/etc/fluent-bit/plugins.conf'
fluentbit::plugins: []

fluentbit::manage_streams_file: true
fluentbit::streams_file: '/etc/td-agent-bit/streams.conf'
fluentbit::streams: []
fluentbit::streams_file: '/etc/fluent-bit/streams.conf'
fluentbit::streams: {}

fluentbit::manage_parsers_file: true
fluentbit::parsers_file: '/etc/td-agent-bit/parsers.conf'
fluentbit::parsers_file: '/etc/fluent-bit/parsers.conf'
fluentbit::parsers:

- name: 'apache'
'apache':
format: 'regex'
regex: '^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$'
time_key: 'time'
time_format: '%d/%b/%Y:%H:%M:%S %z'

- name: 'apache2'
'apache2':
format: 'regex'
regex: '^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$'
time_key: 'time'
time_format: '%d/%b/%Y:%H:%M:%S %z'

- name: 'apache_error'
'apache_error':
format: 'regex'
regex: '^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$'

- name: 'nginx'
'nginx':
format: 'regex'
regex: '^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")'
time_key: 'time'
time_format: '%d/%b/%Y:%H:%M:%S %z'

- name: 'k8s-nginx-ingress'
'k8s-nginx-ingress':
format: 'regex'
regex: '^(?<host>[^ ]*) - \[(?<real_ip>[^ ]*)\] - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<last>[^$]*)'
time_key: 'time'
time_format: '%d/%b/%Y:%H:%M:%S %z'

- name: 'json'
'json':
format: 'json'
time_key: 'time'
time_format: '%d/%b/%Y:%H:%M:%S %z'

- name: 'docker'
'docker':
format: 'json'
time_key: 'time'
time_format: '%Y-%m-%dT%H:%M:%S.%L'
time_keep: true

- name: 'docker-daemon'
'docker-daemon':
format: 'regex'
regex: 'time="(?<time>[^ ]*)" level=(?<level>[^ ]*) msg="(?<msg>[^ ].*)"'
time_key: 'time'
time_format: '%Y-%m-%dT%H:%M:%S.%L'
time_keep: true

- name: 'syslog-rfc5424'
'syslog-rfc5424':
format: 'regex'
regex: '^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$'
time_key: 'time'
time_format: '%Y-%m-%dT%H:%M:%S.%L'
time_keep: true

- name: 'syslog-rfc3164-local'
'syslog-rfc3164-local':
format: 'regex'
regex: '^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$'
regex: '^(?:\<(?<pri>[0-9]+)\>)?(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$'
time_key: 'time'
time_format: '%b %d %H:%M:%S'
time_keep: true

- name: 'syslog-rfc3164'
'syslog-rfc3164':
format: 'regex'
regex: '/^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/'
regex: '^(?:\<(?<pri>[0-9]+)\>)?(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/'
time_key: 'time'
time_format: '%b %d %H:%M:%S'
time_format: '%Y-%m-%dT%H:%M:%S.%L'
time_keep: true

- name: 'mongodb'
'mongodb':
format: 'regex'
regex: '^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$'
time_format: '%Y-%m-%dT%H:%M:%S.%L'
time_keep: true
time_key: 'time'

- name: 'cri'
'cri':
format: 'regex'
regex: '^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$'
time_key: 'time'
time_format: '%Y-%m-%dT%H:%M:%S.%L%z'

- name: 'kube-custom'
'kube-custom':
format: 'regex'
regex: '(?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'

- name: 'cinder'
'cinder':
format: 'regex'
regex: '^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$'
time_format: '%Y-%m-%d %H:%M:%S.%L'
time_keep: false
time_key: 'log_time'

- name: 'glance'
'glance':
format: 'regex'
regex: '^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$'
time_format: '%Y-%m-%d %H:%M:%S.%L'
time_keep: false
time_key: 'log_time'

- name: 'heat'
'heat':
format: 'regex'
regex: '^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$'
time_format: '%Y-%m-%d %H:%M:%S.%L'
time_keep: false
time_key: 'log_time'

- name: 'keystone'
'keystone':
format: 'regex'
regex: '^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$'
time_format: '%Y-%m-%d %H:%M:%S.%L'
time_keep: false
time_key: 'log_time'

- name: 'neutron'
'neutron':
format: 'regex'
regex: '^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$'
time_format: '%Y-%m-%d %H:%M:%S.%L'
time_keep: false
time_key: 'log_time'

- name: 'nova'
'nova':
format: 'regex'
regex: '^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$'
time_format: '%Y-%m-%d %H:%M:%S.%L'
time_keep: false
time_key: 'log_time'

- name: 'crowbar'
'crowbar':
format: 'regex'
regex: '^.*\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+?(?<severity>[^ ]\w+)([\s-]*):?\s+(?<message>.*)'
time_format: '%Y-%m-%dT%H:%M:%S'
time_keep: false
time_key: 'log_time'

- name: 'chefclient'
'chefclient':
format: 'regex'
regex: '^\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+(?<severity>[^ ]\w+):\s+(?<message>.*)$'
time_format: '%Y-%m-%dT%H:%M:%S'
time_keep: false
time_key: 'log_time'

- name: 'mysql_error'
'mysql_error':
format: 'regex'
regex: '^(?<log_time>[^ +][-\d]+[\ T]*[:\dZ]+)\s*(?<myid>[^ ]\d+)\s+\[(?<severity>[^ ]\w+)\](\s+(?<subsystem>[^ ]\w+):){0,1}\s+(?<message>.*)$'
time_format: '%Y-%m-%d %H:%M:%S'
time_keep: false
time_key: 'log_time'

- name: 'mysql_slow'
'mysql_slow':
format: 'regex'
regex: '^# User\@Host:\s+(?<user>[^\@][\w\[\]]+)[@\s]+(?<dbhost>[^ ][-.\w]+)\s+(\[(?<dbhost_address>[.\d]+)\]){0,1}\s+(?<message>.*)$'

- name: 'pacemaker'
'pacemaker':
format: 'regex'
regex: '^\s*(?<log_time>[^ ]* {1,2}[^ ]* [^ ]*) \[(?<pid>\d+)\] (?<node>[\-\w]*)\s*(?<component>\w*):\s+(?<severity>\w+):\s+(?<message>.*)$'
time_format: '%b %d %H:%M:%S'
time_keep: false
time_key: 'log_time'

- name: 'rabbitmq'
'rabbitmq':
format: 'regex'
regex: '^=(?<severity>[^ ]\w+)\s+REPORT[=\s]*(?<log_time>[^ =][-:.\d\w]+)[\s=]+(?<message>.*)$'
time_format: '%d-%b-%Y::%H:%M:%S'
time_keep: false
time_key: 'log_time'

- name: 'universal'
'universal':
format: 'regex'
regex: '^(?<message>.*)$'

- name: 'uuid'
'uuid':
format: 'regex'
regex: '(?<uuid>[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})'

- name: 'ceph'
'ceph':
format: 'regex'
regex: '^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<message>.*)$'
time_format: '%Y-%m-%d %H:%M:%S.%L'
time_keep: false
time_key: 'log_time'

- name: 'java_multiline'
'java_multiline':
format: 'regex'
regex: '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/'
time_key: 'time'
Expand Down
Loading

0 comments on commit d5f9f63

Please sign in to comment.