Skip to content

Commit

Permalink
Adjust ecs@mappings to latest ECS and tests to consider normalizer
Browse files Browse the repository at this point in the history
  • Loading branch information
@eyalkoren
eyalkoren committed Jul 11, 2024
1 parent 1964be5 commit 31d3e17
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 25 deletions.
23 changes: 23 additions & 0 deletions x-pack/plugin/core/template-resources/src/main/resources/[email protected]
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,29 @@
"unmatch_mapping_type": "object"
}
},
{
"ecs_process_fields_with_caseless": {
"mapping": {
"fields": {
"caseless": {
"type": "keyword",
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
},
"type": "keyword"
},
"path_match": [
"process.executable",
"process.*.executable",
"process.name",
"process.*.name"
],
"unmatch_mapping_type": "object"
}
},
{
"ecs_path_match_keyword_and_match_only_text": {
"mapping": {
Expand Down
95 changes: 70 additions & 25 deletions ...ugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {

private static Map<String, Object> ecsDynamicTemplates;
private static Map<String, Map<String, Object>> ecsFlatFieldDefinitions;
private static Map<String, String> ecsFlatMultiFieldDefinitions;
private static Map<String, Map<String, Object>> ecsFlatMultiFieldDefinitions;

@BeforeClass
public static void setupSuiteScopeCluster() throws Exception {
Expand Down Expand Up @@ -142,12 +142,11 @@ private static void prepareEcsDefinitions() throws IOException {
iterator.remove();
}

List<Map<String, String>> multiFields = (List<Map<String, String>>) definitions.get("multi_fields");
List<Map<String, Object>> multiFields = (List<Map<String, Object>>) definitions.get("multi_fields");
if (multiFields != null) {
multiFields.forEach(multiFieldsDefinitions -> {
String subfieldFlatName = Objects.requireNonNull(multiFieldsDefinitions.get("flat_name"));
String subfieldType = Objects.requireNonNull(multiFieldsDefinitions.get("type"));
ecsFlatMultiFieldDefinitions.put(subfieldFlatName, subfieldType);
String subfieldFlatName = (String) Objects.requireNonNull(multiFieldsDefinitions.get("flat_name"));
ecsFlatMultiFieldDefinitions.put(subfieldFlatName, multiFieldsDefinitions);
});
}
}
Expand Down Expand Up @@ -191,7 +190,7 @@ public void testNumericMessage() throws IOException {
verifyEcsMappings(indexName);
}

private void assertType(String expectedType, Map<String, Object> actualMappings) throws IOException {
private void assertType(String expectedType, Map<String, Object> actualMappings) {
assertNotNull("expected to get non-null mappings for field", actualMappings);
assertEquals(expectedType, actualMappings.get("type"));
}
Expand Down Expand Up @@ -406,37 +405,41 @@ private void verifyEcsMappings(String indexName) throws IOException {
Map<String, String> fieldToWrongMappingType = new HashMap<>();
List<String> wronglyIndexedFields = new ArrayList<>();
List<String> wronglyDocValuedFields = new ArrayList<>();
Map<String, String> wronglyNormalizedField2expectedNormalizer = new HashMap<>();
flatFieldMappings.forEach((fieldName, actualMappings) -> {
Map<String, Object> expectedMappings = shallowFieldMapCopy.remove(fieldName);
if (expectedMappings == null) {
nonEcsFields.add(fieldName);
} else {
String expectedType = (String) expectedMappings.get("type");
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
if (expectedMappings.get("index") != actualMappings.get("index")) {
wronglyIndexedFields.add(fieldName);
}
if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
wronglyDocValuedFields.add(fieldName);
}
compareExpectedToActualMappings(
fieldName,
actualMappings,
expectedMappings,
fieldToWrongMappingType,
wronglyIndexedFields,
wronglyDocValuedFields,
wronglyNormalizedField2expectedNormalizer
);
}
});

Map<String, String> shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions);
Map<String, Map<String, Object>> shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions);
logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size());
flatMultiFieldsMappings.forEach((fieldName, actualMappings) -> {
String expectedType = shallowMultiFieldMapCopy.remove(fieldName);
if (expectedType != null) {
Map<String, Object> expectedMultiFieldMappings = shallowMultiFieldMapCopy.remove(fieldName);
if (expectedMultiFieldMappings != null) {
// not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to
// ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead
// to multi-field mapping for ECS fields for which such are not defined
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
compareExpectedToActualMappings(
fieldName,
actualMappings,
expectedMultiFieldMappings,
fieldToWrongMappingType,
wronglyIndexedFields,
wronglyDocValuedFields,
wronglyNormalizedField2expectedNormalizer
);
}
});

Expand All @@ -460,7 +463,11 @@ private void verifyEcsMappings(String indexName) throws IOException {
);
});
fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> {
String ecsExpectedType = (String) ecsFlatFieldDefinitions.get(fieldName).get("type");
Map<String, Object> fieldMappings = ecsFlatFieldDefinitions.get(fieldName);
if (fieldMappings == null) {
fieldMappings = ecsFlatMultiFieldDefinitions.get(fieldName);
}
String ecsExpectedType = (String) fieldMappings.get("type");
logger.error(
"ECS field '{}' should be mapped to type '{}' but is mapped to type '{}'. Update {} accordingly.",
fieldName,
Expand All @@ -472,6 +479,13 @@ private void verifyEcsMappings(String indexName) throws IOException {
nonEcsFields.forEach(field -> logger.error("The test document contains '{}', which is not an ECS field", field));
wronglyIndexedFields.forEach(fieldName -> logger.error("ECS field '{}' should be mapped with \"index: false\"", fieldName));
wronglyDocValuedFields.forEach(fieldName -> logger.error("ECS field '{}' should be mapped with \"doc_values: false\"", fieldName));
wronglyNormalizedField2expectedNormalizer.forEach(
(fieldName, expectedNormalizer) -> logger.error(
"ECS field '{}' should be mapped with \"normalizer: {}\"",
fieldName,
expectedNormalizer
)
);

assertTrue("ECS is not fully covered by the current ECS dynamic templates, see details above", shallowFieldMapCopy.isEmpty());
assertTrue(
Expand All @@ -492,5 +506,36 @@ private void verifyEcsMappings(String indexName) throws IOException {
+ "details above",
wronglyDocValuedFields.isEmpty()
);
assertTrue(
"At least one field was not mapped with the correct normalizer as it should according to its ECS definitions, see "
+ "details above",
wronglyNormalizedField2expectedNormalizer.isEmpty()
);
}

private static void compareExpectedToActualMappings(
String fieldName,
Map<String, Object> actualMappings,
Map<String, Object> expectedMappings,
Map<String, String> fieldToWrongMappingType,
List<String> wronglyIndexedFields,
List<String> wronglyDocValuedFields,
Map<String, String> wronglyNormalizedField2expectedNormalizer
) {
String expectedType = (String) expectedMappings.get("type");
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
if (expectedMappings.get("index") != actualMappings.get("index")) {
wronglyIndexedFields.add(fieldName);
}
if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
wronglyDocValuedFields.add(fieldName);
}
String expectedNormalizer = (String) expectedMappings.get("normalizer");
if (expectedNormalizer != null && expectedNormalizer.equals(actualMappings.get("normalizer")) == false) {
wronglyNormalizedField2expectedNormalizer.put(fieldName, expectedNormalizer);
}
}
}

0 comments on commit 31d3e17

Please sign in to comment.