From 5b4444b8f3e467dd2ad77a881ec69fc2a0d90fee Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 7 Dec 2020 16:13:23 +0100 Subject: [PATCH 1/4] Add data_stream fields to spec --- spec/spec.json | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/spec/spec.json b/spec/spec.json index 3edef7e..452ca7c 100644 --- a/spec/spec.json +++ b/spec/spec.json @@ -82,7 +82,51 @@ "they should set `event.dataset=${service.name}.${appender.name}` if the appender name is available in the logging library.", "Otherwise, agents should also set `event.dataset=${service.name}.log`", "", - "The field helps to filter for different log streams from the same pod, for example and is required for log anomaly detection." + "The field helps to filter for different log streams from the same pod, for example and is required for log anomaly detection.", + "", + "Should have the same value as `data_stream.dataset`.", + "", + "In 8.0, `event.dataset` will be removed in favor of `data_stream.dataset`." + ] + }, + "data_stream.dataset": { + "type": "string", + "required": false, + "url": "https://github.com/elastic/ecs/pull/1145", + "default": "${service.name}.log OR ${service.name}.${appender.name}", + "sanitization": { + "value": { + "replacements": ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", "-"], + "substitute": "_", + "max_chars": 100 + } + }, + "comment": [ + "Configurable by users.", + "Influences which index the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", + "If unspecified, Filebeat will set the value to `default`.", + "", + "Should have the same value as `event.dataset`." + ] + }, + "data_stream.namespace": { + "type": "string", + "required": false, + "url": "https://github.com/elastic/ecs/pull/1145", + "default": null, + "sanitization": { + "value": { + "replacements": ["\\", "/", "*", "?", "\"", "<", ">", "|", " "], + "substitute": "_", + "max_chars": 100 + } + }, + "comment": [ + "Configurable by users.", + "Influences which index the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", + "If unspecified, Filebeat will set the value to `default`.", + "", + "Should have the same value as `event.dataset`." ] }, "process.thread.name": { From 06e80e12c7e8d5c8e346ef6053da0cccd871b6c8 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Mon, 7 Dec 2020 16:26:07 +0100 Subject: [PATCH 2/4] Replace index with data stream --- spec/spec.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/spec.json b/spec/spec.json index 452ca7c..fa517f1 100644 --- a/spec/spec.json +++ b/spec/spec.json @@ -103,7 +103,7 @@ }, "comment": [ "Configurable by users.", - "Influences which index the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", + "Influences which data stream the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", "If unspecified, Filebeat will set the value to `default`.", "", "Should have the same value as `event.dataset`." @@ -123,7 +123,7 @@ }, "comment": [ "Configurable by users.", - "Influences which index the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", + "Influences which data stream the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", "If unspecified, Filebeat will set the value to `default`.", "", "Should have the same value as `event.dataset`." From 9364777b53dfbb582712e6121b7e52824d0d33ae Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 8 Dec 2020 15:20:45 +0100 Subject: [PATCH 3/4] Change default of data_stream.dataset to generic --- spec/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/spec.json b/spec/spec.json index fa517f1..8132c9c 100644 --- a/spec/spec.json +++ b/spec/spec.json @@ -104,7 +104,7 @@ "comment": [ "Configurable by users.", "Influences which data stream the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", - "If unspecified, Filebeat will set the value to `default`.", + "If unspecified, Filebeat will set the value to `generic`.", "", "Should have the same value as `event.dataset`." ] From 8b37b7e2b728df95ae72a47f36fb0c7aca47a65c Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 17 Mar 2021 13:52:49 +0100 Subject: [PATCH 4/4] Add more disallowed characters, require lower case --- spec/spec.json | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/spec/spec.json b/spec/spec.json index 8132c9c..7fd4f55 100644 --- a/spec/spec.json +++ b/spec/spec.json @@ -73,6 +73,14 @@ "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-event.html", "default": "${service.name}.log OR ${service.name}.${appender.name}", + "sanitization": { + "value": { + "replacements": ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":", "-"], + "substitute": "_", + "max_chars": 100, + "lower_case": true + } + }, "comment": [ "Configurable by users.", "If the user manually configures the service name,", @@ -84,21 +92,20 @@ "", "The field helps to filter for different log streams from the same pod, for example and is required for log anomaly detection.", "", - "Should have the same value as `data_stream.dataset`.", - "", - "In 8.0, `event.dataset` will be removed in favor of `data_stream.dataset`." + "Must be in sync with `data_stream.dataset`." ] }, "data_stream.dataset": { "type": "string", "required": false, - "url": "https://github.com/elastic/ecs/pull/1145", + "url": "https://github.com/elastic/ecs/blob/master/rfcs/text/0009-data_stream-fields.md", "default": "${service.name}.log OR ${service.name}.${appender.name}", "sanitization": { "value": { - "replacements": ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", "-"], + "replacements": ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":", "-"], "substitute": "_", - "max_chars": 100 + "max_chars": 100, + "lower_case": true } }, "comment": [ @@ -106,7 +113,7 @@ "Influences which data stream the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", "If unspecified, Filebeat will set the value to `generic`.", "", - "Should have the same value as `event.dataset`." + "Must be in sync with `event.dataset`." ] }, "data_stream.namespace": { @@ -116,9 +123,10 @@ "default": null, "sanitization": { "value": { - "replacements": ["\\", "/", "*", "?", "\"", "<", ">", "|", " "], + "replacements": ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"], "substitute": "_", - "max_chars": 100 + "max_chars": 100, + "lower_case": true } }, "comment": [ @@ -126,7 +134,7 @@ "Influences which data stream the logs will be stored in (`logs-{data_stream.dataset}-{data_stream.namespace}`)", "If unspecified, Filebeat will set the value to `default`.", "", - "Should have the same value as `event.dataset`." + "Must be in sync with `data_stream.dataset`." ] }, "process.thread.name": {