From 08b63c3143e6d40f56218cc8303c7df9e3855349 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 12 Nov 2020 14:50:28 -0600 Subject: [PATCH] [1.x] Remove index parameter for wildcard fields (#1115) (#1119) --- CHANGELOG.md | 2 +- experimental/generated/beats/fields.ecs.yml | 1 - experimental/generated/ecs/ecs_flat.yml | 1 - experimental/generated/ecs/ecs_nested.yml | 1 - schemas/README.md | 5 +++-- scripts/schema/cleaner.py | 3 +++ scripts/tests/unit/test_schema_cleaner.py | 4 ++++ 7 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 956b2a75b..8e77fe6b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ All notable changes to this project will be documented in this file based on the * Added `threat.technique.subtechnique` to capture MITRE ATT&CK® subtechniques. #951 * Added `configuration` as an allowed `event.category`. #963 * Added a new directory with experimental artifacts, which includes all changes - from RFCs that have reached stage 2. #993, #1053 + from RFCs that have reached stage 2. #993, #1053, #1115 #### Improvements diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 26ec99ba2..e94a371c9 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1160,7 +1160,6 @@ norms: false default_field: false description: The stack trace of this error in plain text. - index: true - name: type level: extended type: wildcard diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 7a92b4771..3c14cf04a 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -1599,7 +1599,6 @@ error.stack_trace: dashed_name: error-stack-trace description: The stack trace of this error in plain text. flat_name: error.stack_trace - index: true level: extended multi_fields: - flat_name: error.stack_trace.text diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index da428dae7..f7a2bc93a 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1971,7 +1971,6 @@ error: dashed_name: error-stack-trace description: The stack trace of this error in plain text. flat_name: error.stack_trace - index: true level: extended multi_fields: - flat_name: error.stack_trace.text diff --git a/schemas/README.md b/schemas/README.md index 88440c035..39b18f4bd 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -129,7 +129,8 @@ Supported keys to describe fields Example values that are composite types (array, object) should be quoted to avoid YAML interpretation in ECS-generated artifacts and other downstream projects depending on the schema. - multi\_fields (optional): Specify additional ways to index the field. -- index (optional): If `False`, means field is not indexed (overrides type) +- index (optional): If `False`, means field is not indexed (overrides type). This parameter has no effect + on a `wildcard` field. - format: Field format that can be used in a Kibana index template. - normalize: Normalization steps that should be applied at ingestion time. Supported values: - array: the content of the field should be an array (even when there's only one value). @@ -151,7 +152,7 @@ Supported keys to describe expected values for a field Optionally, entries in this list can specify 'expected\_event\_types'. - expected\_event\_types: list of expected "event.type" values to use in association with that category. - + Supported keys when using the [alias field type](https://www.elastic.co/guide/en/elasticsearch/reference/current/alias.html) ```YAML diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py index ab3acfcae..185d0abed 100644 --- a/scripts/schema/cleaner.py +++ b/scripts/schema/cleaner.py @@ -144,6 +144,9 @@ def field_or_multi_field_datatype_defaults(field_details): field_details.setdefault('ignore_above', 1024) if field_details['type'] == 'text': field_details.setdefault('norms', False) + # wildcard needs the index param stripped + if field_details['type'] == 'wildcard': + field_details.pop('index', None) if 'index' in field_details and not field_details['index']: field_details.setdefault('doc_values', False) diff --git a/scripts/tests/unit/test_schema_cleaner.py b/scripts/tests/unit/test_schema_cleaner.py index 13f78c4e9..bc3dbdc62 100644 --- a/scripts/tests/unit/test_schema_cleaner.py +++ b/scripts/tests/unit/test_schema_cleaner.py @@ -223,6 +223,10 @@ def test_field_defaults(self): cleaner.field_defaults({'field_details': field_details}) self.assertEqual(field_details['doc_values'], False) + field_details = {**field_min_details, **{'type': 'wildcard', 'index': True}} + cleaner.field_defaults({'field_details': field_details}) + self.assertNotIn('index', field_details) + def test_field_defaults_dont_override(self): field_details = { 'description': 'description',