Skip to content

Commit

Permalink
Make ecs@mapping more exhaustive to enforce correct mapping for numer…
Browse files Browse the repository at this point in the history 
…ic and boolean fields as strings
  • Loading branch information
@eyalkoren
eyalkoren committed Sep 21, 2024
1 parent 0697089 commit bd78549
Show file tree
Hide file tree
Showing 2 changed files with 182 additions and 25 deletions.
81 changes: 81 additions & 0 deletions x-pack/plugin/core/template-resources/src/main/resources/[email protected]
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,87 @@
"match_mapping_type": "string"
}
},
{
"ecs_explicit_keywords": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"path_match": [
"*registry.data.bytes"
],
"match_mapping_type": "string"
}
},
{
"ecs_boolean": {
"mapping": {
"type": "boolean"
},
"path_match": [
"*.code_signature.exists",
"*.code_signature.trusted",
"*.code_signature.valid",
"*.go_stripped",
"*.interactive",
"*.same_as_process",
"*faas.coldstart",
"*volume.removable",
"*tls.established",
"*container.security_context.privileged",
"*tls.resumed",
"*process.io.max_bytes_per_process_exceeded",
"*volume.writable"
],
"match_mapping_type": ["boolean", "string"]
}
},
{
"ecs_long": {
"mapping": {
"type": "long"
},
"path_match": [
"*entropy",
"*.size",
"*_size",
"*.as.number",
"*.bytes",
"*bytes_skipped*",
"*.pid",
"*.vpid",
"*.port",
"*.packets",
"*.char_device.major",
"*.char_device.minor",
"*.chi2",
"*.args_count",
"*.virtual_address",
"*.pgid",
"*.thread.id",
"*.header.entrypoint",
"*.uptime",
"*.scanner_stats",
"*.indicator.sightings",
"*process.io.total_bytes_captured",
"*process.tty.columns",
"*event.severity",
"*log.syslog.severity.code",
"*process.parent.exit_code",
"*http.response.status_code",
"*log.syslog.facility.code",
"*process.exit_code",
"*log.origin.file.line",
"*process.tty.rows",
"*event.duration",
"*event.sequence",
"*dns.answers.ttl",
"*log.syslog.priority"

],
"match_mapping_type": ["long", "string"]
}
},
{
"ecs_wildcard": {
"mapping": {
Expand Down
126 changes: 101 additions & 25 deletions ...ugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,10 +208,7 @@ public void testNumericMessage() throws IOException {
}

public void testDateFieldsWithDifferentFormats() throws IOException {
Map<String, Object> dateFieldsMap = ecsFlatFieldDefinitions.entrySet()
.stream()
.filter(entry -> "date".equals(entry.getValue().get("type")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
Map<String, Object> dateFieldsMap = getFieldMappingsByType("date");

// test with iso8601 format
String indexName = "test-date-fields-as-is8601";
Expand All @@ -221,7 +218,7 @@ public void testDateFieldsWithDifferentFormats() throws IOException {
for (String field : dateFieldsMap.keySet()) {
document.put(field, formatter.formatMillis(System.currentTimeMillis()));
}
verifyAllDateFields(indexName, document, dateFieldsMap);
assertFieldsType("date", indexName, document, dateFieldsMap);

// test with milliseconds since epoch format
indexName = "test-date-fields-as-millis";
Expand All @@ -230,22 +227,91 @@ public void testDateFieldsWithDifferentFormats() throws IOException {
for (String field : dateFieldsMap.keySet()) {
document.put(field, System.currentTimeMillis());
}
verifyAllDateFields(indexName, document, dateFieldsMap);
assertFieldsType("date", indexName, document, dateFieldsMap);
}

public void testBooleanFieldsAsString() throws IOException {
Map<String, Object> dateFieldsMap = getFieldMappingsByType("boolean");
String indexName = "test-boolean-fields-as-string";
createTestIndex(indexName);
Map<String, Object> document = new HashMap<>();
for (String field : dateFieldsMap.keySet()) {
document.put(field, Boolean.toString(randomBoolean()));
}
assertFieldsType("boolean", indexName, document, dateFieldsMap);
}

public void testLongFieldsAsString() throws IOException {
Map<String, Object> dateFieldsMap = getFieldMappingsByType("long");
String indexName = "test-long-fields-as-string";
createTestIndex(indexName);
Map<String, Object> document = new HashMap<>();
for (String field : dateFieldsMap.keySet()) {
document.put(field, Long.toString(randomLong()));
}
assertFieldsType("long", indexName, document, dateFieldsMap);
}

private void verifyAllDateFields(String indexName, Map<String, Object> document, Map<String, Object> dateFieldsMap) throws IOException {
public void testFloatFieldsAsString() throws IOException {
Map<String, Object> dateFieldsMap = getFieldMappingsByType("float");
String indexName = "test-flot-fields-as-string";
createTestIndex(indexName);
Map<String, Object> document = new HashMap<>();
for (String field : dateFieldsMap.keySet()) {
document.put(field, Float.toString(randomFloat()));
}
assertFieldsType("float", indexName, document, dateFieldsMap);
}

private static Map<String, Object> getFieldMappingsByType(String type) {
Map<String, Object> ecsBaseFields = ecsFlatFieldDefinitions.entrySet()
.stream()
.filter(entry -> type.equals(entry.getValue().get("type")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

Map<String, Object> completeMapIncludingAttributes = new HashMap<>(ecsBaseFields);
ecsBaseFields.forEach((key, value) -> {
String newKey = "attributes." + key;
completeMapIncludingAttributes.put(newKey, value);
});

return completeMapIncludingAttributes;
}

private void assertFieldsType(String expectedType, String indexName, Map<String, Object> document, Map<String, Object> dateFieldsMap)
throws IOException {
indexDocument(indexName, document);
final Map<String, Object> rawMappings = getMappings(indexName);
final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
final Map<String, String> wronglyTypedFields = new HashMap<>();
flatFieldMappings.forEach((fieldName, fieldMappings) -> {
if (dateFieldsMap.containsKey(fieldName)) {
assertType("date", fieldMappings);
String actualType = (String) fieldMappings.get("type");
if (expectedType.equals(actualType) == false) {
wronglyTypedFields.put(fieldName, actualType);
}
}
});
wronglyTypedFields.forEach((fieldName, actualType) -> {
Object ingestedValue = document.get(fieldName);
logger.error(
"ECS field '{}' is expected to be mapped to '{}', but when provided with a value of type '{}' it gets mapped to '{}'. "
+ "Fix {} accordingly.",
fieldName,
expectedType,
ingestedValue.getClass().getSimpleName().toLowerCase(),
actualType,
ECS_DYNAMIC_TEMPLATES_FILE
);
});
assertTrue(
"At least one field was not mapped with correctly according to its ECS definitions, see details above",
wronglyTypedFields.isEmpty()
);
}

private void assertType(String expectedType, Map<String, Object> actualMappings) {
private void assertFieldType(String expectedType, Map<String, Object> actualMappings) {
assertNotNull("expected to get non-null mappings for field", actualMappings);
assertEquals(expectedType, actualMappings.get("type"));
}
Expand All @@ -266,11 +332,11 @@ public void testUsage() throws IOException {
final Map<String, Object> rawMappings = getMappings(indexName);
final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
assertType("scaled_float", flatFieldMappings.get("host.cpu.usage"));
assertType("scaled_float", flatFieldMappings.get("string.usage"));
assertType("long", flatFieldMappings.get("usage"));
assertType("long", flatFieldMappings.get("root.usage.long"));
assertType("float", flatFieldMappings.get("root.usage.float"));
assertFieldType("scaled_float", flatFieldMappings.get("host.cpu.usage"));
assertFieldType("scaled_float", flatFieldMappings.get("string.usage"));
assertFieldType("long", flatFieldMappings.get("usage"));
assertFieldType("long", flatFieldMappings.get("root.usage.long"));
assertFieldType("float", flatFieldMappings.get("root.usage.float"));
}

public void testOnlyMatchLeafFields() throws IOException {
Expand All @@ -291,14 +357,14 @@ public void testOnlyMatchLeafFields() throws IOException {
final Map<String, Object> rawMappings = getMappings(indexName);
final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
assertType("long", flatFieldMappings.get("foo.message.bar"));
assertType("long", flatFieldMappings.get("foo.url.path.bar"));
assertType("long", flatFieldMappings.get("foo.url.full.bar"));
assertType("long", flatFieldMappings.get("foo.stack_trace.bar"));
assertType("long", flatFieldMappings.get("foo.user_agent.original.bar"));
assertType("long", flatFieldMappings.get("foo.created.bar"));
assertType("float", flatFieldMappings.get("foo._score.bar"));
assertType("long", flatFieldMappings.get("foo.structured_data"));
assertFieldType("long", flatFieldMappings.get("foo.message.bar"));
assertFieldType("long", flatFieldMappings.get("foo.url.path.bar"));
assertFieldType("long", flatFieldMappings.get("foo.url.full.bar"));
assertFieldType("long", flatFieldMappings.get("foo.stack_trace.bar"));
assertFieldType("long", flatFieldMappings.get("foo.user_agent.original.bar"));
assertFieldType("long", flatFieldMappings.get("foo.created.bar"));
assertFieldType("float", flatFieldMappings.get("foo._score.bar"));
assertFieldType("long", flatFieldMappings.get("foo.structured_data"));
}

private static void indexDocument(String indexName, Map<String, Object> flattenedFieldsMap) throws IOException {
Expand Down Expand Up @@ -376,9 +442,6 @@ private Object generateTestValue(String type) {
case "long" -> {
return randomLong();
}
case "int" -> {
return randomInt();
}
case "float", "scaled_float" -> {
return randomFloat();
}
Expand All @@ -401,6 +464,13 @@ private Object generateTestValue(String type) {
// creating multiple subfields
return Map.of("subfield1", randomAlphaOfLength(20), "subfield2", randomAlphaOfLength(20));
}

// TODO - REMOVE ONCE THE ERROR INTRODUCED WITH https://github.com/elastic/ecs/pull/2370 IS FIXED
case "string" -> {
System.err.println("Field type 'string' is not supported by ECS and should be replaced with 'keyword' or 'text'");
return randomAlphaOfLength(20);
}

}
throw new IllegalArgumentException("Unknown field type: " + type);
}
Expand Down Expand Up @@ -576,6 +646,12 @@ private static void compareExpectedToActualMappings(
String expectedType = (String) expectedMappings.get("type");
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {

// TODO - REMOVE ONCE THE ERROR INTRODUCED WITH https://github.com/elastic/ecs/pull/2370 IS FIXED
if (expectedType.equals("string") && actualMappingType.equals("keyword")) {
return;
}

fieldToWrongMappingType.put(fieldName, actualMappingType);
}
if (expectedMappings.get("index") != actualMappings.get("index")) {
Expand Down

0 comments on commit bd78549

Please sign in to comment.