From 7a4f27197d0c0489dd6b4a447830bd8fb1be3518 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Mar 2021 12:28:00 -0500 Subject: [PATCH 1/8] move data_stream fieldset into schema --- {experimental/schemas => schemas}/data_stream.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {experimental/schemas => schemas}/data_stream.yml (100%) diff --git a/experimental/schemas/data_stream.yml b/schemas/data_stream.yml similarity index 100% rename from experimental/schemas/data_stream.yml rename to schemas/data_stream.yml From 563a02d0bc065d0d60955acedc69d797387e7b44 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Mar 2021 12:30:16 -0500 Subject: [PATCH 2/8] artifacts --- code/go/ecs/data_stream.go | 66 ++++++++++++++ docs/field-details.asciidoc | 86 +++++++++++++++++++ docs/fields.asciidoc | 2 + .../generated/elasticsearch/template.json | 6 +- generated/beats/fields.ecs.yml | 52 +++++++++++ generated/csv/fields.csv | 3 + generated/ecs/ecs_flat.yml | 46 ++++++++++ generated/ecs/ecs_nested.yml | 69 +++++++++++++++ generated/elasticsearch/6/template.json | 16 ++++ generated/elasticsearch/7/template.json | 13 +++ .../elasticsearch/component/data_stream.json | 25 ++++++ generated/elasticsearch/template.json | 1 + 12 files changed, 382 insertions(+), 3 deletions(-) create mode 100644 code/go/ecs/data_stream.go create mode 100644 generated/elasticsearch/component/data_stream.json diff --git a/code/go/ecs/data_stream.go b/code/go/ecs/data_stream.go new file mode 100644 index 0000000000..251a97211a --- /dev/null +++ b/code/go/ecs/data_stream.go @@ -0,0 +1,66 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The data_stream fields take part in defining the new data stream naming +// scheme. +// In the new data stream naming scheme the value of the data stream fields +// combine to the name of the actual data stream in the following manner +// `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This +// means the fields can only contain characters that are valid as part of names +// of data streams. More details about this can be found in this +// https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog +// post]. +// An Elasticsearch data stream consists of one or more backing indices, and a +// data stream name forms part of the backing indices names. Due to this +// convention, data streams must also follow index naming restrictions. For +// example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `. +// Please see the Elasticsearch reference for additional +// https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. +type DataStream struct { + // An overarching type for the data stream. + // Currently allowed values are "logs" and "metrics". We expect to also add + // "traces" and "synthetics" in the near future. + Type string `ecs:"type"` + + // The field can contain anything that makes sense to signify the source of + // the data. + // Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data + // streams that otherwise fit, but that do not have dataset set we use the + // value "generic" for the dataset value. `event.dataset` should have the + // same value as `data_stream.dataset`. + // Beyond the Elasticsearch data stream naming criteria noted above, the + // `dataset` value has additional restrictions: + // * Must not contain `-` + // * No longer than 100 characters + Dataset string `ecs:"dataset"` + + // A user defined namespace. Namespaces are useful to allow grouping of + // data. + // Many users already organize their indices this way, and the data stream + // naming scheme now provides this best practice as a default. Many users + // will populate this field with `default`. If no value is used, it falls + // back to `default`. + // Beyond the Elasticsearch index naming criteria noted above, `namespace` + // value has the additional restrictions: + // * Must not contain `-` + // * No longer than 100 characters + Namespace string `ecs:"namespace"` +} diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 2cc663d86e..d5deab753e 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1017,6 +1017,92 @@ example: `docker` |===== +[[ecs-data_stream]] +=== Data Stream Fields + +The data_stream fields take part in defining the new data stream naming scheme. + +In the new data stream naming scheme the value of the data stream fields combine to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. + +An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. + +[discrete] +==== Data Stream Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| +[[field-data-stream-dataset]] +<> + +| The field can contain anything that makes sense to signify the source of the data. + +Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. + +Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: + + * Must not contain `-` + + * No longer than 100 characters + +type: constant_keyword + + + +example: `nginx.access` + +| extended + +// =============================================================== + +| +[[field-data-stream-namespace]] +<> + +| A user defined namespace. Namespaces are useful to allow grouping of data. + +Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. + +Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: + + * Must not contain `-` + + * No longer than 100 characters + +type: constant_keyword + + + +example: `production` + +| extended + +// =============================================================== + +| +[[field-data-stream-type]] +<> + +| An overarching type for the data stream. + +Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. + +type: constant_keyword + + + +example: `logs` + +| extended + +// =============================================================== + +|===== + [[ecs-destination]] === Destination Fields diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc index 7b901e7aa9..a9adf96872 100644 --- a/docs/fields.asciidoc +++ b/docs/fields.asciidoc @@ -32,6 +32,8 @@ all fields are defined. | <> | Fields describing the container that generated this event. +| <> | The data_stream fields take part in defining the new data stream naming scheme. + | <> | Fields about the destination side of a network connection, used with source. | <> | These fields contain information about code libraries dynamically loaded into processes. diff --git a/experimental/generated/elasticsearch/template.json b/experimental/generated/elasticsearch/template.json index e67083a166..55f09b054a 100644 --- a/experimental/generated/elasticsearch/template.json +++ b/experimental/generated/elasticsearch/template.json @@ -9,6 +9,7 @@ "ecs_2.0.0-dev-exp_client", "ecs_2.0.0-dev-exp_cloud", "ecs_2.0.0-dev-exp_container", + "ecs_2.0.0-dev-exp_data_stream", "ecs_2.0.0-dev-exp_destination", "ecs_2.0.0-dev-exp_dll", "ecs_2.0.0-dev-exp_dns", @@ -38,8 +39,7 @@ "ecs_2.0.0-dev-exp_url", "ecs_2.0.0-dev-exp_user", "ecs_2.0.0-dev-exp_user_agent", - "ecs_2.0.0-dev-exp_vulnerability", - "ecs_2.0.0-dev-exp_data_stream" + "ecs_2.0.0-dev-exp_vulnerability" ], "index_patterns": [ "try-ecs-*" @@ -70,4 +70,4 @@ } } } -} \ No newline at end of file +} diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 76d2fed2f3..8cab4052f3 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -634,6 +634,58 @@ ignore_above: 1024 description: Runtime managing this container. example: docker + - name: data_stream + title: Data Stream + group: 2 + description: 'The data_stream fields take part in defining the new data stream + naming scheme. + + In the new data stream naming scheme the value of the data stream fields combine + to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. + This means the fields can only contain characters that are valid as part of + names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog + post]. + + An Elasticsearch data stream consists of one or more backing indices, and a + data stream name forms part of the backing indices names. Due to this convention, + data streams must also follow index naming restrictions. For example, data stream + names cannot include \, /, *, ?, ", <, >, |, ` `. Please see the Elasticsearch + reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + type: group + fields: + - name: dataset + level: extended + type: constant_keyword + description: "The field can contain anything that makes sense to signify the\ + \ source of the data.\nExamples include `nginx.access`, `prometheus`, `endpoint`\ + \ etc. For data streams that otherwise fit, but that do not have dataset set\ + \ we use the value \"generic\" for the dataset value. `event.dataset` should\ + \ have the same value as `data_stream.dataset`.\nBeyond the Elasticsearch\ + \ data stream naming criteria noted above, the `dataset` value has additional\ + \ restrictions:\n * Must not contain `-`\n * No longer than 100 characters" + example: nginx.access + default_field: false + - name: namespace + level: extended + type: constant_keyword + description: "A user defined namespace. Namespaces are useful to allow grouping\ + \ of data.\nMany users already organize their indices this way, and the data\ + \ stream naming scheme now provides this best practice as a default. Many\ + \ users will populate this field with `default`. If no value is used, it falls\ + \ back to `default`.\nBeyond the Elasticsearch index naming criteria noted\ + \ above, `namespace` value has the additional restrictions:\n * Must not\ + \ contain `-`\n * No longer than 100 characters" + example: production + default_field: false + - name: type + level: extended + type: constant_keyword + description: 'An overarching type for the data stream. + + Currently allowed values are "logs" and "metrics". We expect to also add "traces" + and "synthetics" in the near future.' + example: logs + default_field: false - name: destination title: Destination group: 2 diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 9a6affed78..2ba05d75be 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -64,6 +64,9 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 2.0.0-dev,true,container,container.labels,object,extended,,,Image labels. 2.0.0-dev,true,container,container.name,keyword,extended,,,Container name. 2.0.0-dev,true,container,container.runtime,keyword,extended,,docker,Runtime managing this container. +2.0.0-dev,true,data_stream,data_stream.dataset,constant_keyword,extended,,nginx.access,The field can contain anything that makes sense to signify the source of the data. +2.0.0-dev,true,data_stream,data_stream.namespace,constant_keyword,extended,,production,A user defined namespace. Namespaces are useful to allow grouping of data. +2.0.0-dev,true,data_stream,data_stream.type,constant_keyword,extended,,logs,An overarching type for the data stream. 2.0.0-dev,true,destination,destination.address,keyword,extended,,,Destination network address. 2.0.0-dev,true,destination,destination.as.number,long,extended,,15169,Unique number allocated to the autonomous system. 2.0.0-dev,true,destination,destination.as.organization.name,keyword,extended,,Google LLC,Organization name. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 66e5e9fb09..3e8772e2b0 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -756,6 +756,52 @@ container.runtime: normalize: [] short: Runtime managing this container. type: keyword +data_stream.dataset: + dashed_name: data-stream-dataset + description: "The field can contain anything that makes sense to signify the source\ + \ of the data.\nExamples include `nginx.access`, `prometheus`, `endpoint` etc.\ + \ For data streams that otherwise fit, but that do not have dataset set we use\ + \ the value \"generic\" for the dataset value. `event.dataset` should have the\ + \ same value as `data_stream.dataset`.\nBeyond the Elasticsearch data stream naming\ + \ criteria noted above, the `dataset` value has additional restrictions:\n *\ + \ Must not contain `-`\n * No longer than 100 characters" + example: nginx.access + flat_name: data_stream.dataset + level: extended + name: dataset + normalize: [] + short: The field can contain anything that makes sense to signify the source of + the data. + type: constant_keyword +data_stream.namespace: + dashed_name: data-stream-namespace + description: "A user defined namespace. Namespaces are useful to allow grouping\ + \ of data.\nMany users already organize their indices this way, and the data stream\ + \ naming scheme now provides this best practice as a default. Many users will\ + \ populate this field with `default`. If no value is used, it falls back to `default`.\n\ + Beyond the Elasticsearch index naming criteria noted above, `namespace` value\ + \ has the additional restrictions:\n * Must not contain `-`\n * No longer than\ + \ 100 characters" + example: production + flat_name: data_stream.namespace + level: extended + name: namespace + normalize: [] + short: A user defined namespace. Namespaces are useful to allow grouping of data. + type: constant_keyword +data_stream.type: + dashed_name: data-stream-type + description: 'An overarching type for the data stream. + + Currently allowed values are "logs" and "metrics". We expect to also add "traces" + and "synthetics" in the near future.' + example: logs + flat_name: data_stream.type + level: extended + name: type + normalize: [] + short: An overarching type for the data stream. + type: constant_keyword destination.address: dashed_name: destination-address description: 'Some event destination addresses are defined ambiguously. The event diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 9d0461cc38..ec61b44095 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1062,6 +1062,75 @@ container: short: Fields describing the container that generated this event. title: Container type: group +data_stream: + description: 'The data_stream fields take part in defining the new data stream naming + scheme. + + In the new data stream naming scheme the value of the data stream fields combine + to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. + This means the fields can only contain characters that are valid as part of names + of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog + post]. + + An Elasticsearch data stream consists of one or more backing indices, and a data + stream name forms part of the backing indices names. Due to this convention, data + streams must also follow index naming restrictions. For example, data stream names + cannot include \, /, *, ?, ", <, >, |, ` `. Please see the Elasticsearch reference + for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + fields: + data_stream.dataset: + dashed_name: data-stream-dataset + description: "The field can contain anything that makes sense to signify the\ + \ source of the data.\nExamples include `nginx.access`, `prometheus`, `endpoint`\ + \ etc. For data streams that otherwise fit, but that do not have dataset set\ + \ we use the value \"generic\" for the dataset value. `event.dataset` should\ + \ have the same value as `data_stream.dataset`.\nBeyond the Elasticsearch\ + \ data stream naming criteria noted above, the `dataset` value has additional\ + \ restrictions:\n * Must not contain `-`\n * No longer than 100 characters" + example: nginx.access + flat_name: data_stream.dataset + level: extended + name: dataset + normalize: [] + short: The field can contain anything that makes sense to signify the source + of the data. + type: constant_keyword + data_stream.namespace: + dashed_name: data-stream-namespace + description: "A user defined namespace. Namespaces are useful to allow grouping\ + \ of data.\nMany users already organize their indices this way, and the data\ + \ stream naming scheme now provides this best practice as a default. Many\ + \ users will populate this field with `default`. If no value is used, it falls\ + \ back to `default`.\nBeyond the Elasticsearch index naming criteria noted\ + \ above, `namespace` value has the additional restrictions:\n * Must not\ + \ contain `-`\n * No longer than 100 characters" + example: production + flat_name: data_stream.namespace + level: extended + name: namespace + normalize: [] + short: A user defined namespace. Namespaces are useful to allow grouping of + data. + type: constant_keyword + data_stream.type: + dashed_name: data-stream-type + description: 'An overarching type for the data stream. + + Currently allowed values are "logs" and "metrics". We expect to also add "traces" + and "synthetics" in the near future.' + example: logs + flat_name: data_stream.type + level: extended + name: type + normalize: [] + short: An overarching type for the data stream. + type: constant_keyword + group: 2 + name: data_stream + prefix: data_stream. + short: The data_stream fields take part in defining the new data stream naming scheme. + title: Data Stream + type: group destination: description: 'Destination fields capture details about the receiver of a network exchange/packet. These fields are populated from a network event, packet, or other diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json index 0ad8100c0b..2a964ae302 100644 --- a/generated/elasticsearch/6/template.json +++ b/generated/elasticsearch/6/template.json @@ -332,6 +332,22 @@ } } }, + "data_stream": { + "properties": { + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "destination": { "properties": { "address": { diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json index 120dd31501..fea66d43bb 100644 --- a/generated/elasticsearch/7/template.json +++ b/generated/elasticsearch/7/template.json @@ -331,6 +331,19 @@ } } }, + "data_stream": { + "properties": { + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + } + } + }, "destination": { "properties": { "address": { diff --git a/generated/elasticsearch/component/data_stream.json b/generated/elasticsearch/component/data_stream.json new file mode 100644 index 0000000000..f1cf96a92b --- /dev/null +++ b/generated/elasticsearch/component/data_stream.json @@ -0,0 +1,25 @@ +{ + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-data_stream.html", + "ecs_version": "2.0.0-dev" + }, + "template": { + "mappings": { + "properties": { + "data_stream": { + "properties": { + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/generated/elasticsearch/template.json b/generated/elasticsearch/template.json index 24224701c2..933091ab10 100644 --- a/generated/elasticsearch/template.json +++ b/generated/elasticsearch/template.json @@ -9,6 +9,7 @@ "ecs_2.0.0-dev_client", "ecs_2.0.0-dev_cloud", "ecs_2.0.0-dev_container", + "ecs_2.0.0-dev_data_stream", "ecs_2.0.0-dev_destination", "ecs_2.0.0-dev_dll", "ecs_2.0.0-dev_dns", From 752fe57260f942b1f01c38ed9e581e08f47802f0 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Mar 2021 12:35:02 -0500 Subject: [PATCH 3/8] add additional restrictions mentioned in #1304 --- schemas/data_stream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/data_stream.yml b/schemas/data_stream.yml index d651800fa4..af9bc40195 100644 --- a/schemas/data_stream.yml +++ b/schemas/data_stream.yml @@ -11,7 +11,7 @@ this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. - Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `. + Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. fields: From a908e9c30057d07f6f3875b36ee878137d858200 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Mar 2021 12:37:51 -0500 Subject: [PATCH 4/8] updating artifacts --- code/go/ecs/data_stream.go | 4 ++-- docs/field-details.asciidoc | 2 +- experimental/generated/beats/fields.ecs.yml | 4 ++-- experimental/generated/ecs/ecs_nested.yml | 4 ++-- generated/beats/fields.ecs.yml | 4 ++-- generated/ecs/ecs_nested.yml | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/code/go/ecs/data_stream.go b/code/go/ecs/data_stream.go index 251a97211a..36d636458e 100644 --- a/code/go/ecs/data_stream.go +++ b/code/go/ecs/data_stream.go @@ -31,8 +31,8 @@ package ecs // An Elasticsearch data stream consists of one or more backing indices, and a // data stream name forms part of the backing indices names. Due to this // convention, data streams must also follow index naming restrictions. For -// example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `. -// Please see the Elasticsearch reference for additional +// example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, +// or `#`. Please see the Elasticsearch reference for additional // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. type DataStream struct { // An overarching type for the data stream. diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index d5deab753e..1f971d12d8 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1024,7 +1024,7 @@ The data_stream fields take part in defining the new data stream naming scheme. In the new data stream naming scheme the value of the data stream fields combine to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. -An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. +An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. [discrete] ==== Data Stream Field Details diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 99d7c8f951..c43a510a25 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -640,8 +640,8 @@ An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream - names cannot include \, /, *, ?, ", <, >, |, ` `. Please see the Elasticsearch - reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the + Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' type: group fields: - name: dataset diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index ea594fdb32..9598e1d2eb 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1069,8 +1069,8 @@ data_stream: An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names - cannot include \, /, *, ?, ", <, >, |, ` `. Please see the Elasticsearch reference - for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the Elasticsearch + reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' fields: data_stream.dataset: dashed_name: data-stream-dataset diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 8cab4052f3..4ac7bf1f9a 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -649,8 +649,8 @@ An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream - names cannot include \, /, *, ?, ", <, >, |, ` `. Please see the Elasticsearch - reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the + Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' type: group fields: - name: dataset diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index ec61b44095..0fb3f10e48 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1075,8 +1075,8 @@ data_stream: An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names - cannot include \, /, *, ?, ", <, >, |, ` `. Please see the Elasticsearch reference - for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the Elasticsearch + reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' fields: data_stream.dataset: dashed_name: data-stream-dataset From f7ade960e0e2e6618d9144572a517b34cb4c73a0 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Mar 2021 12:52:02 -0500 Subject: [PATCH 5/8] tweak special character formatting --- code/go/ecs/data_stream.go | 5 +++-- docs/field-details.asciidoc | 2 +- experimental/generated/beats/fields.ecs.yml | 4 ++-- experimental/generated/ecs/ecs_nested.yml | 4 ++-- generated/beats/fields.ecs.yml | 4 ++-- generated/ecs/ecs_nested.yml | 4 ++-- schemas/data_stream.yml | 2 +- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/code/go/ecs/data_stream.go b/code/go/ecs/data_stream.go index 36d636458e..44ca62920e 100644 --- a/code/go/ecs/data_stream.go +++ b/code/go/ecs/data_stream.go @@ -31,8 +31,9 @@ package ecs // An Elasticsearch data stream consists of one or more backing indices, and a // data stream name forms part of the backing indices names. Due to this // convention, data streams must also follow index naming restrictions. For -// example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, -// or `#`. Please see the Elasticsearch reference for additional +// example, data stream names cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, +// `|`, ` ` (space character), `,`, or `#`. Please see the Elasticsearch +// reference for additional // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. type DataStream struct { // An overarching type for the data stream. diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 1f971d12d8..927874a1d8 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1024,7 +1024,7 @@ The data_stream fields take part in defining the new data stream naming scheme. In the new data stream naming scheme the value of the data stream fields combine to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. -An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. +An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. [discrete] ==== Data Stream Field Details diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index c43a510a25..6d483b0c1c 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -640,8 +640,8 @@ An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream - names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the - Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + names cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), + `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' type: group fields: - name: dataset diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 9598e1d2eb..4b72ea6c6c 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1069,8 +1069,8 @@ data_stream: An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names - cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the Elasticsearch - reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), + `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' fields: data_stream.dataset: dashed_name: data-stream-dataset diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 4ac7bf1f9a..a2cf11845f 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -649,8 +649,8 @@ An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream - names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the - Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + names cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), + `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' type: group fields: - name: dataset diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 0fb3f10e48..bdd44d8048 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1075,8 +1075,8 @@ data_stream: An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names - cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. Please see the Elasticsearch - reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' + cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), + `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].' fields: data_stream.dataset: dashed_name: data-stream-dataset diff --git a/schemas/data_stream.yml b/schemas/data_stream.yml index af9bc40195..d89726318d 100644 --- a/schemas/data_stream.yml +++ b/schemas/data_stream.yml @@ -11,7 +11,7 @@ this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. - Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include \, /, *, ?, ", <, >, |, ` `, `,`, or `#`. + Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. fields: From 72bbe292b83145624c603b2b813ab93a3c3a9769 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Mar 2021 13:32:52 -0500 Subject: [PATCH 6/8] add missing colon --- code/go/ecs/data_stream.go | 2 +- docs/field-details.asciidoc | 2 +- experimental/generated/beats/fields.ecs.yml | 2 +- experimental/generated/ecs/ecs_nested.yml | 2 +- generated/beats/fields.ecs.yml | 2 +- generated/ecs/ecs_nested.yml | 2 +- schemas/data_stream.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/code/go/ecs/data_stream.go b/code/go/ecs/data_stream.go index 44ca62920e..e641fbb514 100644 --- a/code/go/ecs/data_stream.go +++ b/code/go/ecs/data_stream.go @@ -22,7 +22,7 @@ package ecs // The data_stream fields take part in defining the new data stream naming // scheme. // In the new data stream naming scheme the value of the data stream fields -// combine to the name of the actual data stream in the following manner +// combine to the name of the actual data stream in the following manner: // `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This // means the fields can only contain characters that are valid as part of names // of data streams. More details about this can be found in this diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 927874a1d8..49b6d9e88b 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1022,7 +1022,7 @@ example: `docker` The data_stream fields take part in defining the new data stream naming scheme. -In the new data stream naming scheme the value of the data stream fields combine to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. +In the new data stream naming scheme the value of the data stream fields combine to the name of the actual data stream in the following manner: `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 6d483b0c1c..f2c68b783f 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -632,7 +632,7 @@ naming scheme. In the new data stream naming scheme the value of the data stream fields combine - to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. + to the name of the actual data stream in the following manner: `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 4b72ea6c6c..12c1be0458 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1061,7 +1061,7 @@ data_stream: scheme. In the new data stream naming scheme the value of the data stream fields combine - to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. + to the name of the actual data stream in the following manner: `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index a2cf11845f..3067eab765 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -641,7 +641,7 @@ naming scheme. In the new data stream naming scheme the value of the data stream fields combine - to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. + to the name of the actual data stream in the following manner: `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index bdd44d8048..2c76a378f1 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1067,7 +1067,7 @@ data_stream: scheme. In the new data stream naming scheme the value of the data stream fields combine - to the name of the actual data stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. + to the name of the actual data stream in the following manner: `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. diff --git a/schemas/data_stream.yml b/schemas/data_stream.yml index d89726318d..de6f89ba45 100644 --- a/schemas/data_stream.yml +++ b/schemas/data_stream.yml @@ -6,7 +6,7 @@ The data_stream fields take part in defining the new data stream naming scheme. In the new data stream naming scheme the value of the data stream fields combine to the name of the actual data - stream in the following manner `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields + stream in the following manner: `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post]. From 3212361dedaeebd4c1557d6a107c271e1922743b Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 18 Mar 2021 13:38:41 -0500 Subject: [PATCH 7/8] changelog --- CHANGELOG.next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 3a0c7e7476..58eafff020 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -16,6 +16,7 @@ Thanks, you're awesome :-) --> #### Added +* Add `data_stream` fieldset. #1307 * Add `orchestrator` fieldset as beta fields. #1326 * Extend `threat.*` experimental fields with proposed changes from RFC 0018. #1344, #1351 From 8cf619760f67bcd11f998199ab66f143cc8b5467 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Mon, 19 Apr 2021 10:17:45 -0500 Subject: [PATCH 8/8] add beta attribute for fieldset --- docs/field-details.asciidoc | 2 ++ experimental/generated/ecs/ecs_nested.yml | 1 + experimental/generated/elasticsearch/template.json | 2 +- generated/ecs/ecs_nested.yml | 1 + schemas/data_stream.yml | 2 ++ 5 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 49b6d9e88b..52ed080c6d 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1026,6 +1026,8 @@ In the new data stream naming scheme the value of the data stream fields combine An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions]. +beta::[ These fields are in beta and are subject to change.] + [discrete] ==== Data Stream Field Details diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 12c1be0458..be5465f8fd 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1057,6 +1057,7 @@ container: title: Container type: group data_stream: + beta: These fields are in beta and are subject to change. description: 'The data_stream fields take part in defining the new data stream naming scheme. diff --git a/experimental/generated/elasticsearch/template.json b/experimental/generated/elasticsearch/template.json index 55f09b054a..5775a87263 100644 --- a/experimental/generated/elasticsearch/template.json +++ b/experimental/generated/elasticsearch/template.json @@ -70,4 +70,4 @@ } } } -} +} \ No newline at end of file diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 2c76a378f1..b0958cccce 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1063,6 +1063,7 @@ container: title: Container type: group data_stream: + beta: These fields are in beta and are subject to change. description: 'The data_stream fields take part in defining the new data stream naming scheme. diff --git a/schemas/data_stream.yml b/schemas/data_stream.yml index de6f89ba45..a169b0a61c 100644 --- a/schemas/data_stream.yml +++ b/schemas/data_stream.yml @@ -2,6 +2,8 @@ - name: data_stream title: Data Stream short: The data_stream fields take part in defining the new data stream naming scheme. + beta: > + These fields are in beta and are subject to change. description: > The data_stream fields take part in defining the new data stream naming scheme.