From 1b81b78cb13e1b1924ca82841e6fb4d903c3ce09 Mon Sep 17 00:00:00 2001 From: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com> Date: Fri, 13 Jan 2023 09:32:53 -0600 Subject: [PATCH] Match timestamps without year in pipeline tests with regex (#4984) - Timestamps that are parsed without a year (such as those on BSD-style syslog messages) will have their expected values inherit the year the expected files are generated in. This means that tests will only pass in the year that the expected files are generated. - The relevant timestamp field (@timestamp, for example) has been added to the pipeline test config as a dynamic field, and a regex pattern is used to match the expected format of the timestamp. --- .../data_stream/log/_dev/test/pipeline/test-common-config.yml | 2 ++ .../data_stream/log/_dev/test/pipeline/test-common-config.yml | 2 ++ .../data_stream/log/_dev/test/pipeline/test-common-config.yml | 1 + .../data_stream/log/_dev/test/pipeline/test-common-config.yml | 2 ++ .../data_stream/log/_dev/test/pipeline/test-common-config.yml | 2 ++ .../data_stream/log/_dev/test/pipeline/test-common-config.yml | 3 +++ .../data_stream/log/_dev/test/pipeline/test-common-config.yml | 2 ++ .../data_stream/log/_dev/test/pipeline/test-common-config.yml | 2 ++ .../alerts/_dev/test/pipeline/test-common-config.yml | 2 ++ 9 files changed, 18 insertions(+) diff --git a/packages/cisco_aironet/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/cisco_aironet/data_stream/log/_dev/test/pipeline/test-common-config.yml index 4da22641654..0f37e513074 100644 --- a/packages/cisco_aironet/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/cisco_aironet/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,3 +1,5 @@ +dynamic_fields: + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - preserve_original_event diff --git a/packages/cisco_ftd/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/cisco_ftd/data_stream/log/_dev/test/pipeline/test-common-config.yml index 4da22641654..0f37e513074 100644 --- a/packages/cisco_ftd/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/cisco_ftd/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,3 +1,5 @@ +dynamic_fields: + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - preserve_original_event diff --git a/packages/cisco_ios/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/cisco_ios/data_stream/log/_dev/test/pipeline/test-common-config.yml index 5622947e4b8..7aaa32ce9c8 100644 --- a/packages/cisco_ios/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/cisco_ios/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,5 +1,6 @@ dynamic_fields: event.ingested: ".*" + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - preserve_original_event diff --git a/packages/cisco_ise/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/cisco_ise/data_stream/log/_dev/test/pipeline/test-common-config.yml index 4da22641654..0f37e513074 100644 --- a/packages/cisco_ise/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/cisco_ise/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,3 +1,5 @@ +dynamic_fields: + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - preserve_original_event diff --git a/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-config.yml index 4da22641654..0f37e513074 100644 --- a/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,3 +1,5 @@ +dynamic_fields: + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - preserve_original_event diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-common-config.yml index 4da22641654..3f773a2f98d 100644 --- a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,3 +1,6 @@ +dynamic_fields: + "event.created": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - preserve_original_event diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-common-config.yml index d2bf5ce8251..ff08a016252 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,3 +1,5 @@ +dynamic_fields: + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - preserve_original_event diff --git a/packages/symantec_endpoint/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/symantec_endpoint/data_stream/log/_dev/test/pipeline/test-common-config.yml index 00d4f1de2be..23ea0594777 100644 --- a/packages/symantec_endpoint/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/symantec_endpoint/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,3 +1,5 @@ +dynamic_fields: + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - forwarded diff --git a/packages/zscaler_zia/data_stream/alerts/_dev/test/pipeline/test-common-config.yml b/packages/zscaler_zia/data_stream/alerts/_dev/test/pipeline/test-common-config.yml index 4da22641654..0f37e513074 100644 --- a/packages/zscaler_zia/data_stream/alerts/_dev/test/pipeline/test-common-config.yml +++ b/packages/zscaler_zia/data_stream/alerts/_dev/test/pipeline/test-common-config.yml @@ -1,3 +1,5 @@ +dynamic_fields: + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}" fields: tags: - preserve_original_event