Add common event format plugin

CHANGELOG.md

@@ -5,11 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
 ## [0.0.83] Unreleased
 
 ### Added
 
 - Add `tcp` and `udp` plugin ([PR341](https://github.com/observIQ/stanza-plugins/pull/341))
+- Added `common_event_format` plugin ([328](https://github.com/observIQ/stanza-plugins/pull/328))
 
 ### Fixed
 
@@ -23,7 +25,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.0.81] 2021-09-28
 
-### Fixed
 - HAProxy: Resolved an issue where http logs using default format can fail to parse ([PR342](https://github.com/observIQ/stanza-plugins/pull/342))
 
 ## [0.0.80] 2021-09-23

plugins/common_event_format.yaml

@@ -0,0 +1,178 @@
+# Plugin Info
+version: 0.0.1
+title: Common Event Format Log Parser
+description: File Input Common Event Format Parser
+min_stanza_version: 1.2.7
+supported_platforms:
+  - linux
+  - windows
+  - kubernetes
+  - macos
+parameters:
+  - name: file_log_path
+    label: File Path
+    description: Specify a single path or multiple paths to read one or many files. You may also use a wildcard (*) to read multiple files within a directory.
+    type: strings
+  - name: exclude_file_log_path
+    label: Exclude File Path
+    description: Specify a single path or multiple paths to exclude one or many files from being read. You may also use a wildcard (*) to exclude multiple files from being read within a directory.
+    type: strings
+    default: []
+    advanced_config: true
+  - name: log_type
+    label: Type
+    description: Adds the specified 'Type' as a label to each log message.
+    type: string
+    default: json
+  - name: location
+    label: Geographic Location
+    description: The geographic location (timezone) to use when parsing logs that contain a timestamp
+    type: enum
+    valid_values: ["Africa/Abidjan","Africa/Accra","Africa/Addis_Ababa","Africa/Algiers","Africa/Asmara","Africa/Bamako","Africa/Bangui","Africa/Banjul","Africa/Bissau","Africa/Blantyre","Africa/Brazzaville","Africa/Bujumbura","Africa/Cairo","Africa/Casablanca","Africa/Ceuta","Africa/Conakry","Africa/Dakar","Africa/Dar_es_Salaam","Africa/Djibouti","Africa/Douala","Africa/El_Aaiun","Africa/Freetown","Africa/Gaborone","Africa/Harare","Africa/Johannesburg","Africa/Juba","Africa/Kampala","Africa/Khartoum","Africa/Kigali","Africa/Kinshasa","Africa/Lagos","Africa/Libreville","Africa/Lome","Africa/Luanda","Africa/Lubumbashi","Africa/Lusaka","Africa/Malabo","Africa/Maputo","Africa/Maseru","Africa/Mbabane","Africa/Mogadishu","Africa/Monrovia","Africa/Nairobi","Africa/Ndjamena","Africa/Niamey","Africa/Nouakchott","Africa/Ouagadougou","Africa/Porto-Novo","Africa/Sao_Tome","Africa/Tripoli","Africa/Tunis","Africa/Windhoek","America/Adak","America/Anchorage","America/Anguilla","America/Antigua","America/Araguaina","America/Argentina/Buenos_Aires","America/Argentina/Catamarca","America/Argentina/Cordoba","America/Argentina/Jujuy","America/Argentina/La_Rioja","America/Argentina/Mendoza","America/Argentina/Rio_Gallegos","America/Argentina/Salta","America/Argentina/San_Juan","America/Argentina/San_Luis","America/Argentina/Tucuman","America/Argentina/Ushuaia","America/Aruba","America/Asuncion","America/Atikokan","America/Bahia","America/Bahia_Banderas","America/Barbados","America/Belem","America/Belize","America/Blanc-Sablon","America/Boa_Vista","America/Bogota","America/Boise","America/Cambridge_Bay","America/Campo_Grande","America/Cancun","America/Caracas","America/Cayenne","America/Cayman","America/Chicago","America/Chihuahua","America/Costa_Rica","America/Creston","America/Cuiaba","America/Curacao","America/Danmarkshavn","America/Dawson","America/Dawson_Creek","America/Denver","America/Detroit","America/Dominica","America/Edmonton","America/Eirunepe","America/El_Salvador","America/Fort_Nelson","America/Fortaleza","America/Glace_Bay","America/Goose_Bay","America/Grand_Turk","America/Grenada","America/Guadeloupe","America/Guatemala","America/Guayaquil","America/Guyana","America/Halifax","America/Havana","America/Hermosillo","America/Indiana/Indianapolis","America/Indiana/Knox","America/Indiana/Marengo","America/Indiana/Petersburg","America/Indiana/Tell_City","America/Indiana/Vevay","America/Indiana/Vincennes","America/Indiana/Winamac","America/Inuvik","America/Iqaluit","America/Jamaica","America/Juneau","America/Kentucky/Louisville","America/Kentucky/Monticello","America/Kralendijk","America/La_Paz","America/Lima","America/Los_Angeles","America/Lower_Princes","America/Maceio","America/Managua","America/Manaus","America/Marigot","America/Martinique","America/Matamoros","America/Mazatlan","America/Menominee","America/Merida","America/Metlakatla","America/Mexico_City","America/Miquelon","America/Moncton","America/Monterrey","America/Montevideo","America/Montserrat","America/Nassau","America/New_York","America/Nipigon","America/Nome","America/Noronha","America/North_Dakota/Beulah","America/North_Dakota/Center","America/North_Dakota/New_Salem","America/Nuuk","America/Ojinaga","America/Panama","America/Pangnirtung","America/Paramaribo","America/Phoenix","America/Port-au-Prince","America/Port_of_Spain","America/Porto_Velho","America/Puerto_Rico","America/Punta_Arenas","America/Rainy_River","America/Rankin_Inlet","America/Recife","America/Regina","America/Resolute","America/Rio_Branco","America/Santarem","America/Santiago","America/Santo_Domingo","America/Sao_Paulo","America/Scoresbysund","America/Sitka","America/St_Barthelemy","America/St_Johns","America/St_Kitts","America/St_Lucia","America/St_Thomas","America/St_Vincent","America/Swift_Current","America/Tegucigalpa","America/Thule","America/Thunder_Bay","America/Tijuana","America/Toronto","America/Tortola","America/Vancouver","America/Whitehorse","America/Winnipeg","America/Yakutat","America/Yellowknife","Antarctica/Casey","Antarctica/Davis","Antarctica/DumontDUrville","Antarctica/Macquarie","Antarctica/Mawson","Antarctica/McMurdo","Antarctica/Palmer","Antarctica/Rothera","Antarctica/Syowa","Antarctica/Troll","Antarctica/Vostok","Arctic/Longyearbyen","Asia/Aden","Asia/Almaty","Asia/Amman","Asia/Anadyr","Asia/Aqtau","Asia/Aqtobe","Asia/Ashgabat","Asia/Atyrau","Asia/Baghdad","Asia/Bahrain","Asia/Baku","Asia/Bangkok","Asia/Barnaul","Asia/Beirut","Asia/Bishkek","Asia/Brunei","Asia/Chita","Asia/Choibalsan","Asia/Colombo","Asia/Damascus","Asia/Dhaka","Asia/Dili","Asia/Dubai","Asia/Dushanbe","Asia/Famagusta","Asia/Gaza","Asia/Hebron","Asia/Ho_Chi_Minh","Asia/Hong_Kong","Asia/Hovd","Asia/Irkutsk","Asia/Jakarta","Asia/Jayapura","Asia/Jerusalem","Asia/Kabul","Asia/Kamchatka","Asia/Karachi","Asia/Kathmandu","Asia/Khandyga","Asia/Kolkata","Asia/Krasnoyarsk","Asia/Kuala_Lumpur","Asia/Kuching","Asia/Kuwait","Asia/Macau","Asia/Magadan","Asia/Makassar","Asia/Manila","Asia/Muscat","Asia/Nicosia","Asia/Novokuznetsk","Asia/Novosibirsk","Asia/Omsk","Asia/Oral","Asia/Phnom_Penh","Asia/Pontianak","Asia/Pyongyang","Asia/Qatar","Asia/Qostanay","Asia/Qyzylorda","Asia/Riyadh","Asia/Sakhalin","Asia/Samarkand","Asia/Seoul","Asia/Shanghai","Asia/Singapore","Asia/Srednekolymsk","Asia/Taipei","Asia/Tashkent","Asia/Tbilisi","Asia/Tehran","Asia/Thimphu","Asia/Tokyo","Asia/Tomsk","Asia/Ulaanbaatar","Asia/Urumqi","Asia/Ust-Nera","Asia/Vientiane","Asia/Vladivostok","Asia/Yakutsk","Asia/Yangon","Asia/Yekaterinburg","Asia/Yerevan","Atlantic/Azores","Atlantic/Bermuda","Atlantic/Canary","Atlantic/Cape_Verde","Atlantic/Faroe","Atlantic/Madeira","Atlantic/Reykjavik","Atlantic/South_Georgia","Atlantic/St_Helena","Atlantic/Stanley","Australia/Adelaide","Australia/Brisbane","Australia/Broken_Hill","Australia/Currie","Australia/Darwin","Australia/Eucla","Australia/Hobart","Australia/Lindeman","Australia/Lord_Howe","Australia/Melbourne","Australia/Perth","Australia/Sydney","Europe/Amsterdam","Europe/Andorra","Europe/Astrakhan","Europe/Athens","Europe/Belgrade","Europe/Berlin","Europe/Bratislava","Europe/Brussels","Europe/Bucharest","Europe/Budapest","Europe/Busingen","Europe/Chisinau","Europe/Copenhagen","Europe/Dublin","Europe/Gibraltar","Europe/Guernsey","Europe/Helsinki","Europe/Isle_of_Man","Europe/Istanbul","Europe/Jersey","Europe/Kaliningrad","Europe/Kiev","Europe/Kirov","Europe/Lisbon","Europe/Ljubljana","Europe/London","Europe/Luxembourg","Europe/Madrid","Europe/Malta","Europe/Mariehamn","Europe/Minsk","Europe/Monaco","Europe/Moscow","Europe/Oslo","Europe/Paris","Europe/Podgorica","Europe/Prague","Europe/Riga","Europe/Rome","Europe/Samara","Europe/San_Marino","Europe/Sarajevo","Europe/Saratov","Europe/Simferopol","Europe/Skopje","Europe/Sofia","Europe/Stockholm","Europe/Tallinn","Europe/Tirane","Europe/Ulyanovsk","Europe/Uzhgorod","Europe/Vaduz","Europe/Vatican","Europe/Vienna","Europe/Vilnius","Europe/Volgograd","Europe/Warsaw","Europe/Zagreb","Europe/Zaporozhye","Europe/Zurich","Indian/Antananarivo","Indian/Chagos","Indian/Christmas","Indian/Cocos","Indian/Comoro","Indian/Kerguelen","Indian/Mahe","Indian/Maldives","Indian/Mauritius","Indian/Mayotte","Indian/Reunion","Pacific/Apia","Pacific/Auckland","Pacific/Bougainville","Pacific/Chatham","Pacific/Chuuk","Pacific/Easter","Pacific/Efate","Pacific/Enderbury","Pacific/Fakaofo","Pacific/Fiji","Pacific/Funafuti","Pacific/Galapagos","Pacific/Gambier","Pacific/Guadalcanal","Pacific/Guam","Pacific/Honolulu","Pacific/Kiritimati","Pacific/Kosrae","Pacific/Kwajalein","Pacific/Majuro","Pacific/Marquesas","Pacific/Midway","Pacific/Nauru","Pacific/Niue","Pacific/Norfolk","Pacific/Noumea","Pacific/Pago_Pago","Pacific/Palau","Pacific/Pitcairn","Pacific/Pohnpei","Pacific/Port_Moresby","Pacific/Rarotonga","Pacific/Saipan","Pacific/Tahiti","Pacific/Tarawa","Pacific/Tongatapu","Pacific/Wake","Pacific/Wallis","UTC"]
+    default: "UTC"
+  - name: start_at
+    label: Start At
+    description: Start reading file from 'beginning' or 'end'
+    type: enum
+    valid_values:
+      - beginning
+      - end
+    default: end
+# Set Defaults
+# {{$log_type := default "cef" .log_type}}
+# {{$start_at := default "end" .start_at}}
+# {{$location := default "UTC" .location}}
+
+# Pipeline Template
+pipeline:
+  - type: file_input
+    start_at: '{{ $start_at }}'
+    include:
+# {{ range $i, $fp := .file_log_path  }}
+      - '{{ $fp }}'
+# {{ end }}
+# {{ if .exclude_file_log_path }}
+    exclude:
+  # {{ range $i, $efp := .exclude_file_log_path  }}
+      - '{{ $efp }}'
+  # {{ end }}
+# {{ end }}
+    encoding: 'utf-8'
+    include_file_name: true
+
+  - id: add_plugin_id
+    type: add
+    field: '$labels.plugin_id'
+    value: 'common_event_format'
+
+  - id: add_log_type
+    type: add
+    field: '$labels.log_type'
+    value: '{{ $log_type }}'
+
+  # Router to check for syslog prefix
+  - id: timestamp
+    type: router
+    default: cef_parser_no_syslog_prefix
+    routes:
+      - expr: '$record matches "^\\w{3}\\s+\\d{1,2}\\s+\\d{2}:\\d{2}:\\d{2}"'
+        output: timestamp_parser
+
+  # Timestamp Jan 02 15:04:05 - Parse syslog prefix. 
+  # If more than one timestamp parser is required then it will need to be move to a stand alone timestamp_parser
+  - id: timestamp_parser
+    type: regex_parser
+    parse_from: $record
+    regex: '^(?P<timestamp>\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+((?P<hostname>[^\s]+)\s+)?(?P<cef_headers>[\d\D]+)'
+    timestamp:
+      parse_from: timestamp
+      layout_type: gotime
+      layout: 'Jan 02 15:04:05'
+      location: {{$location}}
+
+  # Parse CEF log entry with syslog prefix
+  # CEF:<Version>|<Device Vendor>|<Device Product>|<Device Version>|<Signature ID>|<Name>|<Severity>|<Extensions>
+  - id: cef_parser_syslog_prefix
+    type: csv_parser
+    if: '$record.cef_headers != nil'
+    parse_from: $record.cef_headers
+    header: 'version|device_vendor|device_product|device_version|signature_id|name|severity|extensions'
+    header_delimiter: "|"
+    delimiter: "|"
+    output: version_parser
+
+  # Parse CEF log entry without syslog prefix
+  - id: cef_parser_no_syslog_prefix
+    type: csv_parser
+    header: 'version|device_vendor|device_product|device_version|signature_id|name|severity|extensions'
+    header_delimiter: "|"
+    delimiter: "|"
+    output: version_parser
+
+  # Parse version as integer from version field.
+  - id: version_parser
+    type: regex_parser
+    if: '$record.version != nil and $record.version matches "CEF:[^\\|]*"'
+    regex: 'CEF:(?P<version>[^\|]*)'
+    parse_from: $record.version
+
+  - id: promote_version
+    type: move
+    if: '$record.version != nil'
+    from: $record.version
+    to: '$labels["version"]'
+
+  - id: promote_device_product
+    type: move
+    if: '$record.device_product != nil'
+    from: $record.device_product
+    to: '$labels["device"]'
+
+  - id: promote_hostname
+    type: move
+    if: '$record.hostname != nil'
+    from: $record.hostname
+    to: '$resource["hostname"]'
+
+  - id: promote_device_vendor
+    type: move
+    if: '$record.device_vendor != nil'
+    from: $record.device_vendor
+    to: '$resource["device_vendor"]'
+
+  - id: promote_device_version
+    type: move
+    if: '$record.device_version != nil'
+    from: $record.device_version
+    to: '$resource["device_version"]'
+
+  - id: move_name
+    type: move
+    if: '$record.name != nil'
+    from: $record.name
+    to: $record.message
+
+  # Parse severity using CEF severity standards.
+  # Currently not putting unknown which is in standard in parser so it will be assigned default.
+  - type: severity_parser
+    if: '$record.severity != nil'
+    parse_from: $record.severity
+    preset: none
+    mapping:
+      info:
+        - min: 1
+          max: 3
+        - low
+      warning:
+        - min: 4
+          max: 6
+        - medium
+      error:
+        - min: 7
+          max: 8
+        - high
+      critical:
+        - min: 9
+          max: 10
+        - very-high
+    output: {{ .output }}

schemas/common_event_format.yaml

@@ -0,0 +1,45 @@
+version: str()
+title: str()
+description: str()
+parameters: list(include('parameter'))
+pipeline: list(include('operator'))
+---
+parameter:
+  name: str()
+  label: str()
+  description: str()
+  type: str()
+  valid_values: list(required=False)
+
+operator:
+  id: str(required=False)
+  type: str()
+  include: list(required=False)
+  start_at: str(required=False)
+  labels: map(str(), required=False)
+  output: str(required=False) 
+  regex: str(required=False)
+  timestamp: map(include('timestamp_list'), str(), map(), required=False)
+  severity: map(include('severity_list'), str(), map(), required=False)
+
+timestamp_list:
+  parse_from: str()
+  layout: str()
+
+severity_list:
+  parse_from: str()
+  preset: str(required=False)
+  mapping: list(include('mapping_list'), str(), required=False)
+  preserve_to: str(required=False)
+  if: str(required=False)
+
+mapping_list:
+  info: str(required=False)
+  error: str(required=False)
+  warning: str(required=False)
+  critical: str(required=False)
+  debug: str(required=False)
+  trace: str(required=False)
+  alert: str(required=False)
+  emergency: str(required=False)
+  catastrophe: str(required=False)

test/configs/common_event_format/invalid/invaild_file_log_path.yaml

@@ -0,0 +1,4 @@
+pipeline:
+- type: common_event_format
+  file_log_path: "/var/log/json.log"
+- type: stdout

test/configs/common_event_format/invalid/location.yaml

@@ -0,0 +1,6 @@
+pipeline:
+- type: common_event_format
+  file_log_path: 
+    - "/var/log/json.log"
+  location: nowhere
+- type: stdout

test/configs/common_event_format/valid/minimal.yaml

@@ -0,0 +1,5 @@
+pipeline:
+- type: common_event_format
+  file_log_path: 
+    - "/var/log/cef.log"
+- type: stdout