Skip to content

Commit

Permalink
Full coverage of ECS by ecs@mappings when date_detection is disabled (e…
Browse files Browse the repository at this point in the history 
…lastic#112444)

(cherry picked from commit 86a3eff)

# Conflicts:
#	x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
#	x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
  • Loading branch information
@eyalkoren
eyalkoren committed Sep 9, 2024
1 parent d68a094 commit f9e2bbc
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 29 deletions.
6 changes: 6 additions & 0 deletions docs/changelog/112444.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 112444
summary: Full coverage of ECS by ecs@mappings when `date_detection` is disabled
area: Mapping
type: bug
issues:
- 112398
6 changes: 5 additions & 1 deletion x-pack/plugin/core/template-resources/src/main/resources/[email protected]
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,11 @@
"ingested",
"*.ingested",
"*.start",
"*.end"
"*.end",
"*.indicator.first_seen",
"*.indicator.last_seen",
"*.indicator.modified_at",
"*threat.enrichments.matched.occurred"
],
"unmatch_mapping_type": "object"
}
Expand Down
146 changes: 119 additions & 27 deletions ...ugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.FormatNames;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
Expand Down Expand Up @@ -63,7 +64,7 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {

private static Map<String, Object> ecsDynamicTemplates;
private static Map<String, Map<String, Object>> ecsFlatFieldDefinitions;
private static Map<String, String> ecsFlatMultiFieldDefinitions;
private static Map<String, Map<String, Object>> ecsFlatMultiFieldDefinitions;

@BeforeClass
public static void setupSuiteScopeCluster() throws Exception {
Expand Down Expand Up @@ -142,12 +143,11 @@ private static void prepareEcsDefinitions() throws IOException {
iterator.remove();
}

List<Map<String, String>> multiFields = (List<Map<String, String>>) definitions.get("multi_fields");
List<Map<String, Object>> multiFields = (List<Map<String, Object>>) definitions.get("multi_fields");
if (multiFields != null) {
multiFields.forEach(multiFieldsDefinitions -> {
String subfieldFlatName = Objects.requireNonNull(multiFieldsDefinitions.get("flat_name"));
String subfieldType = Objects.requireNonNull(multiFieldsDefinitions.get("type"));
ecsFlatMultiFieldDefinitions.put(subfieldFlatName, subfieldType);
String subfieldFlatName = (String) Objects.requireNonNull(multiFieldsDefinitions.get("flat_name"));
ecsFlatMultiFieldDefinitions.put(subfieldFlatName, multiFieldsDefinitions);
});
}
}
Expand All @@ -166,6 +166,22 @@ public void testFlattenedFields() throws IOException {
verifyEcsMappings(indexName);
}

public void testFlattenedFieldsWithinAttributes() throws IOException {
String indexName = "test-flattened-attributes";
createTestIndex(indexName);
Map<String, Object> flattenedFieldsMap = createTestDocument(true);
indexDocument(indexName, Map.of("attributes", flattenedFieldsMap));
verifyEcsMappings(indexName, "attributes.");
}

public void testFlattenedFieldsWithinResourceAttributes() throws IOException {
String indexName = "test-flattened-attributes";
createTestIndex(indexName);
Map<String, Object> flattenedFieldsMap = createTestDocument(true);
indexDocument(indexName, Map.of("resource.attributes", flattenedFieldsMap));
verifyEcsMappings(indexName, "resource.attributes.");
}

public void testFlattenedFieldsWithoutSubobjects() throws IOException {
String indexName = "test_flattened_fields_subobjects_false";
createTestIndex(indexName, Map.of("subobjects", false));
Expand All @@ -191,7 +207,45 @@ public void testNumericMessage() throws IOException {
verifyEcsMappings(indexName);
}

private void assertType(String expectedType, Map<String, Object> actualMappings) throws IOException {
public void testDateFieldsWithDifferentFormats() throws IOException {
Map<String, Object> dateFieldsMap = ecsFlatFieldDefinitions.entrySet()
.stream()
.filter(entry -> "date".equals(entry.getValue().get("type")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

// test with iso8601 format
String indexName = "test-date-fields-as-is8601";
createTestIndex(indexName);
Map<String, Object> document = new HashMap<>();
DateFormatter formatter = DateFormatter.forPattern(FormatNames.ISO8601.getName());
for (String field : dateFieldsMap.keySet()) {
document.put(field, formatter.formatMillis(System.currentTimeMillis()));
}
verifyAllDateFields(indexName, document, dateFieldsMap);

// test with milliseconds since epoch format
indexName = "test-date-fields-as-millis";
createTestIndex(indexName);
document = new HashMap<>();
for (String field : dateFieldsMap.keySet()) {
document.put(field, System.currentTimeMillis());
}
verifyAllDateFields(indexName, document, dateFieldsMap);
}

private void verifyAllDateFields(String indexName, Map<String, Object> document, Map<String, Object> dateFieldsMap) throws IOException {
indexDocument(indexName, document);
final Map<String, Object> rawMappings = getMappings(indexName);
final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
flatFieldMappings.forEach((fieldName, fieldMappings) -> {
if (dateFieldsMap.containsKey(fieldName)) {
assertType("date", fieldMappings);
}
});
}

private void assertType(String expectedType, Map<String, Object> actualMappings) {
assertNotNull("expected to get non-null mappings for field", actualMappings);
assertEquals(expectedType, actualMappings.get("type"));
}
Expand Down Expand Up @@ -297,6 +351,7 @@ private static void createTestIndex(String indexName, @Nullable Map<String, Obje
} else {
indexMappings = ecsDynamicTemplates;
}
indexMappings.put("date_detection", false);
try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) {
bodyBuilder.startObject();
bodyBuilder.startObject("settings");
Expand Down Expand Up @@ -334,7 +389,7 @@ private Object generateTestValue(String type) {
return "test";
}
case "date" -> {
return DateFormatter.forPattern("strict_date_optional_time").formatMillis(System.currentTimeMillis());
return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).formatMillis(System.currentTimeMillis());
}
case "ip" -> {
return NetworkAddress.format(randomIp(true));
Expand Down Expand Up @@ -395,12 +450,19 @@ private void processRawMappingsSubtree(
}

private void verifyEcsMappings(String indexName) throws IOException {
verifyEcsMappings(indexName, "");
}

private void verifyEcsMappings(String indexName, String fieldPrefix) throws IOException {
final Map<String, Object> rawMappings = getMappings(indexName);
final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
final Map<String, Map<String, Object>> flatMultiFieldsMappings = new HashMap<>();
processRawMappingsSubtree(rawMappings, flatFieldMappings, flatMultiFieldsMappings, "");

Map<String, Map<String, Object>> shallowFieldMapCopy = new HashMap<>(ecsFlatFieldDefinitions);
Map<String, Map<String, Object>> shallowFieldMapCopy = ecsFlatFieldDefinitions.entrySet()
.stream()
.collect(Collectors.toMap(e -> fieldPrefix + e.getKey(), Map.Entry::getValue));

logger.info("Testing mapping of {} ECS fields", shallowFieldMapCopy.size());
List<String> nonEcsFields = new ArrayList<>();
Map<String, String> fieldToWrongMappingType = new HashMap<>();
Expand All @@ -411,32 +473,35 @@ private void verifyEcsMappings(String indexName) throws IOException {
if (expectedMappings == null) {
nonEcsFields.add(fieldName);
} else {
String expectedType = (String) expectedMappings.get("type");
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
if (expectedMappings.get("index") != actualMappings.get("index")) {
wronglyIndexedFields.add(fieldName);
}
if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
wronglyDocValuedFields.add(fieldName);
}
compareExpectedToActualMappings(
fieldName,
actualMappings,
expectedMappings,
fieldToWrongMappingType,
wronglyIndexedFields,
wronglyDocValuedFields
);
}
});

Map<String, String> shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions);
Map<String, Map<String, Object>> shallowMultiFieldMapCopy = ecsFlatMultiFieldDefinitions.entrySet()
.stream()
.collect(Collectors.toMap(e -> fieldPrefix + e.getKey(), Map.Entry::getValue));
logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size());
flatMultiFieldsMappings.forEach((fieldName, actualMappings) -> {
String expectedType = shallowMultiFieldMapCopy.remove(fieldName);
if (expectedType != null) {
Map<String, Object> expectedMultiFieldMappings = shallowMultiFieldMapCopy.remove(fieldName);
if (expectedMultiFieldMappings != null) {
// not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to
// ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead
// to multi-field mapping for ECS fields for which such are not defined
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
compareExpectedToActualMappings(
fieldName,
actualMappings,
expectedMultiFieldMappings,
fieldToWrongMappingType,
wronglyIndexedFields,
wronglyDocValuedFields
);
}
});

Expand All @@ -460,7 +525,13 @@ private void verifyEcsMappings(String indexName) throws IOException {
);
});
fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> {
String ecsExpectedType = (String) ecsFlatFieldDefinitions.get(fieldName).get("type");
// if fieldPrefix is not null, we need to remove it from the field name for the ECS lookup
String ecsFieldName = fieldPrefix == null ? fieldName : fieldName.substring(fieldPrefix.length());
Map<String, Object> fieldMappings = ecsFlatFieldDefinitions.get(ecsFieldName);
if (fieldMappings == null) {
fieldMappings = ecsFlatMultiFieldDefinitions.get(ecsFieldName);
}
String ecsExpectedType = (String) fieldMappings.get("type");
logger.error(
"ECS field '{}' should be mapped to type '{}' but is mapped to type '{}'. Update {} accordingly.",
fieldName,
Expand Down Expand Up @@ -493,4 +564,25 @@ private void verifyEcsMappings(String indexName) throws IOException {
wronglyDocValuedFields.isEmpty()
);
}

private static void compareExpectedToActualMappings(
String fieldName,
Map<String, Object> actualMappings,
Map<String, Object> expectedMappings,
Map<String, String> fieldToWrongMappingType,
List<String> wronglyIndexedFields,
List<String> wronglyDocValuedFields
) {
String expectedType = (String) expectedMappings.get("type");
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
if (expectedMappings.get("index") != actualMappings.get("index")) {
wronglyIndexedFields.add(fieldName);
}
if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
wronglyDocValuedFields.add(fieldName);
}
}
}
2 changes: 1 addition & 1 deletion x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {

// The stack template registry version. This number must be incremented when we make changes
// to built-in templates.
public static final int REGISTRY_VERSION = 12;
public static final int REGISTRY_VERSION = 14;

public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version";
public static final Setting<Boolean> STACK_TEMPLATES_ENABLED = Setting.boolSetting(
Expand Down

0 comments on commit f9e2bbc

Please sign in to comment.