From 7fb13c5a5115dcb0545a9c693433f5a0063ed444 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 15 Jun 2020 09:07:45 -0500 Subject: [PATCH] Explicitly set ECS version in Filebeat modules. - Add test to check if ecs.version is set - add_fields to azure/activitylogs - add_fields to azure/auditlogs - add_fields to azure/signinlogs - add_fields to checkpoint/firewall - add_fields to crowdstrike/falcon - add_fields to fortinet/firewall - add_fields to traefik/access Relates #17688 --- CHANGELOG.next.asciidoc | 1 + .../traefik/access/config/traefik-access.yml | 5 +++ filebeat/tests/system/test_modules.py | 2 ++ libbeat/tests/system/beat/beat.py | 32 +++++++++++++++++++ .../activitylogs/config/azure-eventhub.yml | 5 +++ .../module/azure/activitylogs/config/file.yml | 6 ++++ .../azure/auditlogs/config/azure-eventhub.yml | 5 +++ .../module/azure/auditlogs/config/file.yml | 5 +++ .../signinlogs/config/azure-eventhub.yml | 5 +++ .../module/azure/signinlogs/config/file.yml | 5 +++ .../checkpoint/firewall/config/firewall.yml | 4 +++ .../crowdstrike/falcon/config/falcon.yml | 4 +++ .../fortinet/firewall/config/firewall.yml | 4 +++ 13 files changed, 83 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 93da0220287..30f8a2f088c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -397,6 +397,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - The s3 input can now automatically detect gzipped objects. {issue}18283[18283] {pull}18764[18764] - Add geoip AS lookup & improve ECS categorization in aws cloudtrail fileset. {issue}18644[18644] {pull}18958[18958] - Improved performance of PANW sample dashboards. {issue}19031[19031] {pull}19032[19032] +- Explicitly set ECS version in all Filebeat modules. {pull}19198[19198] *Heartbeat* diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index 0afd17317d4..183de629867 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -4,3 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 81dde391c11..c62bad7a213 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -109,6 +109,8 @@ def test_fileset_file(self, module, fileset, test_file): def run_on_file(self, module, fileset, test_file, cfgfile): print("Testing {}/{} on {}".format(module, fileset, test_file)) + self.assert_explicit_ecs_version_set(module, fileset) + try: self.es.indices.delete(index=self.index_name) except: diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 13e81746dcf..67a2fa179f9 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -11,6 +11,7 @@ import yaml import hashlib import re +import glob from datetime import datetime, timedelta from .compose import ComposeMixin @@ -722,3 +723,34 @@ def get_beat_version(self): proc.wait() return self.get_log_lines(logfile="version")[0].split()[2] + + def assert_explicit_ecs_version_set(self, module, fileset): + """ + Assert that the module explicitly sets the ECS version field. + """ + def get_config_paths(modules_path, module, fileset): + pathname = os.path.abspath(modules_path + + "/" + + module + + "/" + + fileset + + "/" + + "config/*.yml") + return glob.glob(pathname) + + def is_ecs_version_set(path): + # parsing the yml file would be better but go templates in + # the file make that difficult + with open(path) as fhandle: + for line in fhandle: + if re.search("ecs\.version", line): + return True + return False + + errors = [] + for cfg_path in get_config_paths(self.modules_path, module, fileset): + if not is_ecs_version_set(cfg_path): + errors.append("{}".format(cfg_path)) + if len(errors) > 0: + raise Exception("{}/{} ecs.version not explicitly set in:\n{}".format(module, fileset, '\n'.join(errors))) + diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index 9f24f85e3eb..8c4c42d60cf 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -8,3 +8,8 @@ resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/activitylogs/config/file.yml b/x-pack/filebeat/module/azure/activitylogs/config/file.yml index 456cc5dce7c..498a7906457 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/file.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/file.yml @@ -6,3 +6,9 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index f8b88d18a4a..239e7ef2249 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -7,3 +7,8 @@ storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/file.yml b/x-pack/filebeat/module/azure/auditlogs/config/file.yml index 456cc5dce7c..937446eb523 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/file.yml @@ -6,3 +6,8 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index f8b88d18a4a..239e7ef2249 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -7,3 +7,8 @@ storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/file.yml b/x-pack/filebeat/module/azure/signinlogs/config/file.yml index 456cc5dce7c..937446eb523 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/file.yml @@ -6,3 +6,8 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml index 637a2899315..12440f8fffe 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml @@ -20,3 +20,7 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml index 9f5de43be3b..689bd725530 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml +++ b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml @@ -20,3 +20,7 @@ processors: lang: javascript id: crowdstrike_falcon file: ${path.home}/module/crowdstrike/falcon/config/pipeline.js +- add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml index 3cf11a126a0..725ad622fa0 100644 --- a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml @@ -24,3 +24,7 @@ tags: {{.tags}} processors: - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.5.0