diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json index 1951431859ffe..2a04d6fb4b589 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json @@ -109,6 +109,29 @@ "unmatch_mapping_type": "object" } }, + { + "ecs_process_fields_with_caseless": { + "mapping": { + "fields": { + "caseless": { + "type": "keyword", + "normalizer": "lowercase" + }, + "text": { + "type": "match_only_text" + } + }, + "type": "keyword" + }, + "path_match": [ + "process.executable", + "process.*.executable", + "process.name", + "process.*.name" + ], + "unmatch_mapping_type": "object" + } + }, { "ecs_path_match_keyword_and_match_only_text": { "mapping": { diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java index 8bdf7b30a9997..b52b91a79c7e4 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -63,7 +63,7 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase { private static Map ecsDynamicTemplates; private static Map> ecsFlatFieldDefinitions; - private static Map ecsFlatMultiFieldDefinitions; + private static Map> ecsFlatMultiFieldDefinitions; @BeforeClass public static void setupSuiteScopeCluster() throws Exception { @@ -142,12 +142,11 @@ private static void prepareEcsDefinitions() throws IOException { iterator.remove(); } - List> multiFields = (List>) definitions.get("multi_fields"); + List> multiFields = (List>) definitions.get("multi_fields"); if (multiFields != null) { multiFields.forEach(multiFieldsDefinitions -> { - String subfieldFlatName = Objects.requireNonNull(multiFieldsDefinitions.get("flat_name")); - String subfieldType = Objects.requireNonNull(multiFieldsDefinitions.get("type")); - ecsFlatMultiFieldDefinitions.put(subfieldFlatName, subfieldType); + String subfieldFlatName = (String) Objects.requireNonNull(multiFieldsDefinitions.get("flat_name")); + ecsFlatMultiFieldDefinitions.put(subfieldFlatName, multiFieldsDefinitions); }); } } @@ -191,7 +190,7 @@ public void testNumericMessage() throws IOException { verifyEcsMappings(indexName); } - private void assertType(String expectedType, Map actualMappings) throws IOException { + private void assertType(String expectedType, Map actualMappings) { assertNotNull("expected to get non-null mappings for field", actualMappings); assertEquals(expectedType, actualMappings.get("type")); } @@ -406,37 +405,41 @@ private void verifyEcsMappings(String indexName) throws IOException { Map fieldToWrongMappingType = new HashMap<>(); List wronglyIndexedFields = new ArrayList<>(); List wronglyDocValuedFields = new ArrayList<>(); + Map wronglyNormalizedField2expectedNormalizer = new HashMap<>(); flatFieldMappings.forEach((fieldName, actualMappings) -> { Map expectedMappings = shallowFieldMapCopy.remove(fieldName); if (expectedMappings == null) { nonEcsFields.add(fieldName); } else { - String expectedType = (String) expectedMappings.get("type"); - String actualMappingType = (String) actualMappings.get("type"); - if (actualMappingType.equals(expectedType) == false) { - fieldToWrongMappingType.put(fieldName, actualMappingType); - } - if (expectedMappings.get("index") != actualMappings.get("index")) { - wronglyIndexedFields.add(fieldName); - } - if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) { - wronglyDocValuedFields.add(fieldName); - } + compareExpectedToActualMappings( + fieldName, + actualMappings, + expectedMappings, + fieldToWrongMappingType, + wronglyIndexedFields, + wronglyDocValuedFields, + wronglyNormalizedField2expectedNormalizer + ); } }); - Map shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions); + Map> shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions); logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size()); flatMultiFieldsMappings.forEach((fieldName, actualMappings) -> { - String expectedType = shallowMultiFieldMapCopy.remove(fieldName); - if (expectedType != null) { + Map expectedMultiFieldMappings = shallowMultiFieldMapCopy.remove(fieldName); + if (expectedMultiFieldMappings != null) { // not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to // ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead // to multi-field mapping for ECS fields for which such are not defined - String actualMappingType = (String) actualMappings.get("type"); - if (actualMappingType.equals(expectedType) == false) { - fieldToWrongMappingType.put(fieldName, actualMappingType); - } + compareExpectedToActualMappings( + fieldName, + actualMappings, + expectedMultiFieldMappings, + fieldToWrongMappingType, + wronglyIndexedFields, + wronglyDocValuedFields, + wronglyNormalizedField2expectedNormalizer + ); } }); @@ -460,7 +463,11 @@ private void verifyEcsMappings(String indexName) throws IOException { ); }); fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> { - String ecsExpectedType = (String) ecsFlatFieldDefinitions.get(fieldName).get("type"); + Map fieldMappings = ecsFlatFieldDefinitions.get(fieldName); + if (fieldMappings == null) { + fieldMappings = ecsFlatMultiFieldDefinitions.get(fieldName); + } + String ecsExpectedType = (String) fieldMappings.get("type"); logger.error( "ECS field '{}' should be mapped to type '{}' but is mapped to type '{}'. Update {} accordingly.", fieldName, @@ -472,6 +479,13 @@ private void verifyEcsMappings(String indexName) throws IOException { nonEcsFields.forEach(field -> logger.error("The test document contains '{}', which is not an ECS field", field)); wronglyIndexedFields.forEach(fieldName -> logger.error("ECS field '{}' should be mapped with \"index: false\"", fieldName)); wronglyDocValuedFields.forEach(fieldName -> logger.error("ECS field '{}' should be mapped with \"doc_values: false\"", fieldName)); + wronglyNormalizedField2expectedNormalizer.forEach( + (fieldName, expectedNormalizer) -> logger.error( + "ECS field '{}' should be mapped with \"normalizer: {}\"", + fieldName, + expectedNormalizer + ) + ); assertTrue("ECS is not fully covered by the current ECS dynamic templates, see details above", shallowFieldMapCopy.isEmpty()); assertTrue( @@ -492,5 +506,36 @@ private void verifyEcsMappings(String indexName) throws IOException { + "details above", wronglyDocValuedFields.isEmpty() ); + assertTrue( + "At least one field was not mapped with the correct normalizer as it should according to its ECS definitions, see " + + "details above", + wronglyNormalizedField2expectedNormalizer.isEmpty() + ); + } + + private static void compareExpectedToActualMappings( + String fieldName, + Map actualMappings, + Map expectedMappings, + Map fieldToWrongMappingType, + List wronglyIndexedFields, + List wronglyDocValuedFields, + Map wronglyNormalizedField2expectedNormalizer + ) { + String expectedType = (String) expectedMappings.get("type"); + String actualMappingType = (String) actualMappings.get("type"); + if (actualMappingType.equals(expectedType) == false) { + fieldToWrongMappingType.put(fieldName, actualMappingType); + } + if (expectedMappings.get("index") != actualMappings.get("index")) { + wronglyIndexedFields.add(fieldName); + } + if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) { + wronglyDocValuedFields.add(fieldName); + } + String expectedNormalizer = (String) expectedMappings.get("normalizer"); + if (expectedNormalizer != null && expectedNormalizer.equals(actualMappings.get("normalizer")) == false) { + wronglyNormalizedField2expectedNormalizer.put(fieldName, expectedNormalizer); + } } }