diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json index 997bd942881ad..a7703af84f78e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json @@ -60,6 +60,87 @@ "match_mapping_type": "string" } }, + { + "ecs_explicit_keywords": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "path_match": [ + "*registry.data.bytes" + ], + "match_mapping_type": "string" + } + }, + { + "ecs_boolean": { + "mapping": { + "type": "boolean" + }, + "path_match": [ + "*.code_signature.exists", + "*.code_signature.trusted", + "*.code_signature.valid", + "*.go_stripped", + "*.interactive", + "*.same_as_process", + "*faas.coldstart", + "*volume.removable", + "*tls.established", + "*container.security_context.privileged", + "*tls.resumed", + "*process.io.max_bytes_per_process_exceeded", + "*volume.writable" + ], + "match_mapping_type": ["boolean", "string"] + } + }, + { + "ecs_long": { + "mapping": { + "type": "long" + }, + "path_match": [ + "*entropy", + "*.size", + "*_size", + "*.as.number", + "*.bytes", + "*bytes_skipped*", + "*.pid", + "*.vpid", + "*.port", + "*.packets", + "*.char_device.major", + "*.char_device.minor", + "*.chi2", + "*.args_count", + "*.virtual_address", + "*.pgid", + "*.thread.id", + "*.header.entrypoint", + "*.uptime", + "*.scanner_stats", + "*.indicator.sightings", + "*process.io.total_bytes_captured", + "*process.tty.columns", + "*event.severity", + "*log.syslog.severity.code", + "*process.parent.exit_code", + "*http.response.status_code", + "*log.syslog.facility.code", + "*process.exit_code", + "*log.origin.file.line", + "*process.tty.rows", + "*event.duration", + "*event.sequence", + "*dns.answers.ttl", + "*log.syslog.priority" + + ], + "match_mapping_type": ["long", "string"] + } + }, { "ecs_wildcard": { "mapping": { diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java index eba239cd97e28..211bad1280076 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -208,10 +208,7 @@ public void testNumericMessage() throws IOException { } public void testDateFieldsWithDifferentFormats() throws IOException { - Map dateFieldsMap = ecsFlatFieldDefinitions.entrySet() - .stream() - .filter(entry -> "date".equals(entry.getValue().get("type"))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map dateFieldsMap = getFieldMappingsByType("date"); // test with iso8601 format String indexName = "test-date-fields-as-is8601"; @@ -221,7 +218,7 @@ public void testDateFieldsWithDifferentFormats() throws IOException { for (String field : dateFieldsMap.keySet()) { document.put(field, formatter.formatMillis(System.currentTimeMillis())); } - verifyAllDateFields(indexName, document, dateFieldsMap); + assertFieldsType("date", indexName, document, dateFieldsMap); // test with milliseconds since epoch format indexName = "test-date-fields-as-millis"; @@ -230,22 +227,91 @@ public void testDateFieldsWithDifferentFormats() throws IOException { for (String field : dateFieldsMap.keySet()) { document.put(field, System.currentTimeMillis()); } - verifyAllDateFields(indexName, document, dateFieldsMap); + assertFieldsType("date", indexName, document, dateFieldsMap); + } + + public void testBooleanFieldsAsString() throws IOException { + Map dateFieldsMap = getFieldMappingsByType("boolean"); + String indexName = "test-boolean-fields-as-string"; + createTestIndex(indexName); + Map document = new HashMap<>(); + for (String field : dateFieldsMap.keySet()) { + document.put(field, Boolean.toString(randomBoolean())); + } + assertFieldsType("boolean", indexName, document, dateFieldsMap); + } + + public void testLongFieldsAsString() throws IOException { + Map dateFieldsMap = getFieldMappingsByType("long"); + String indexName = "test-long-fields-as-string"; + createTestIndex(indexName); + Map document = new HashMap<>(); + for (String field : dateFieldsMap.keySet()) { + document.put(field, Long.toString(randomLong())); + } + assertFieldsType("long", indexName, document, dateFieldsMap); } - private void verifyAllDateFields(String indexName, Map document, Map dateFieldsMap) throws IOException { + public void testFloatFieldsAsString() throws IOException { + Map dateFieldsMap = getFieldMappingsByType("float"); + String indexName = "test-flot-fields-as-string"; + createTestIndex(indexName); + Map document = new HashMap<>(); + for (String field : dateFieldsMap.keySet()) { + document.put(field, Float.toString(randomFloat())); + } + assertFieldsType("float", indexName, document, dateFieldsMap); + } + + private static Map getFieldMappingsByType(String type) { + Map ecsBaseFields = ecsFlatFieldDefinitions.entrySet() + .stream() + .filter(entry -> type.equals(entry.getValue().get("type"))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map completeMapIncludingAttributes = new HashMap<>(ecsBaseFields); + ecsBaseFields.forEach((key, value) -> { + String newKey = "attributes." + key; + completeMapIncludingAttributes.put(newKey, value); + }); + + return completeMapIncludingAttributes; + } + + private void assertFieldsType(String expectedType, String indexName, Map document, Map dateFieldsMap) + throws IOException { indexDocument(indexName, document); final Map rawMappings = getMappings(indexName); final Map> flatFieldMappings = new HashMap<>(); processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), ""); + final Map wronglyTypedFields = new HashMap<>(); flatFieldMappings.forEach((fieldName, fieldMappings) -> { if (dateFieldsMap.containsKey(fieldName)) { - assertType("date", fieldMappings); + String actualType = (String) fieldMappings.get("type"); + if (expectedType.equals(actualType) == false) { + wronglyTypedFields.put(fieldName, actualType); + } } }); + wronglyTypedFields.forEach((fieldName, actualType) -> { + Object ingestedValue = document.get(fieldName); + logger.error( + "ECS field '{}' is expected to be mapped to '{}', but when provided with a value of type '{}' it gets mapped to '{}'. " + + "Fix {} accordingly.", + fieldName, + expectedType, + ingestedValue.getClass().getSimpleName().toLowerCase(), + actualType, + ECS_DYNAMIC_TEMPLATES_FILE + ); + }); + assertTrue( + "At least one field was not mapped with correctly according to its ECS definitions, see details above", + wronglyTypedFields.isEmpty() + ); } - private void assertType(String expectedType, Map actualMappings) { + private void assertFieldType(String expectedType, Map actualMappings) { assertNotNull("expected to get non-null mappings for field", actualMappings); assertEquals(expectedType, actualMappings.get("type")); } @@ -266,11 +332,11 @@ public void testUsage() throws IOException { final Map rawMappings = getMappings(indexName); final Map> flatFieldMappings = new HashMap<>(); processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), ""); - assertType("scaled_float", flatFieldMappings.get("host.cpu.usage")); - assertType("scaled_float", flatFieldMappings.get("string.usage")); - assertType("long", flatFieldMappings.get("usage")); - assertType("long", flatFieldMappings.get("root.usage.long")); - assertType("float", flatFieldMappings.get("root.usage.float")); + assertFieldType("scaled_float", flatFieldMappings.get("host.cpu.usage")); + assertFieldType("scaled_float", flatFieldMappings.get("string.usage")); + assertFieldType("long", flatFieldMappings.get("usage")); + assertFieldType("long", flatFieldMappings.get("root.usage.long")); + assertFieldType("float", flatFieldMappings.get("root.usage.float")); } public void testOnlyMatchLeafFields() throws IOException { @@ -291,14 +357,14 @@ public void testOnlyMatchLeafFields() throws IOException { final Map rawMappings = getMappings(indexName); final Map> flatFieldMappings = new HashMap<>(); processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), ""); - assertType("long", flatFieldMappings.get("foo.message.bar")); - assertType("long", flatFieldMappings.get("foo.url.path.bar")); - assertType("long", flatFieldMappings.get("foo.url.full.bar")); - assertType("long", flatFieldMappings.get("foo.stack_trace.bar")); - assertType("long", flatFieldMappings.get("foo.user_agent.original.bar")); - assertType("long", flatFieldMappings.get("foo.created.bar")); - assertType("float", flatFieldMappings.get("foo._score.bar")); - assertType("long", flatFieldMappings.get("foo.structured_data")); + assertFieldType("long", flatFieldMappings.get("foo.message.bar")); + assertFieldType("long", flatFieldMappings.get("foo.url.path.bar")); + assertFieldType("long", flatFieldMappings.get("foo.url.full.bar")); + assertFieldType("long", flatFieldMappings.get("foo.stack_trace.bar")); + assertFieldType("long", flatFieldMappings.get("foo.user_agent.original.bar")); + assertFieldType("long", flatFieldMappings.get("foo.created.bar")); + assertFieldType("float", flatFieldMappings.get("foo._score.bar")); + assertFieldType("long", flatFieldMappings.get("foo.structured_data")); } private static void indexDocument(String indexName, Map flattenedFieldsMap) throws IOException { @@ -376,9 +442,6 @@ private Object generateTestValue(String type) { case "long" -> { return randomLong(); } - case "int" -> { - return randomInt(); - } case "float", "scaled_float" -> { return randomFloat(); } @@ -401,6 +464,13 @@ private Object generateTestValue(String type) { // creating multiple subfields return Map.of("subfield1", randomAlphaOfLength(20), "subfield2", randomAlphaOfLength(20)); } + + // TODO - REMOVE ONCE THE ERROR INTRODUCED WITH https://github.com/elastic/ecs/pull/2370 IS FIXED + case "string" -> { + System.err.println("Field type 'string' is not supported by ECS and should be replaced with 'keyword' or 'text'"); + return randomAlphaOfLength(20); + } + } throw new IllegalArgumentException("Unknown field type: " + type); } @@ -576,6 +646,12 @@ private static void compareExpectedToActualMappings( String expectedType = (String) expectedMappings.get("type"); String actualMappingType = (String) actualMappings.get("type"); if (actualMappingType.equals(expectedType) == false) { + + // TODO - REMOVE ONCE THE ERROR INTRODUCED WITH https://github.com/elastic/ecs/pull/2370 IS FIXED + if (expectedType.equals("string") && actualMappingType.equals("keyword")) { + return; + } + fieldToWrongMappingType.put(fieldName, actualMappingType); } if (expectedMappings.get("index") != actualMappings.get("index")) {