From 7d2a5a4f7aac8ff0a90e10b0939fe9c5cdf98cf5 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Fri, 8 Dec 2023 11:58:36 +0800 Subject: [PATCH 01/11] Add unmatch_mapping_type; support array of types Add an unmatch_mapping_type condition to dynamic templates, and add support for specifying a list of types to match_mapping_type. --- .../index/mapper/DynamicTemplate.java | 100 +++++++++++++----- .../mapper/DynamicTemplateParseTests.java | 20 ++++ .../index/mapper/DynamicTemplatesTests.java | 35 +++++- 3 files changed, 126 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index cc37391d982b3..cd20293e40986 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -18,10 +18,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Stream; @@ -257,7 +260,8 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe List pathUnmatch = new ArrayList<>(4); Map mapping = null; boolean runtime = false; - String matchMappingType = null; + Object matchMappingType = null; + Object unmatchMappingType = null; String matchPattern = MatchType.DEFAULT.toString(); for (Map.Entry entry : conf.entrySet()) { @@ -271,7 +275,9 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe } else if ("path_unmatch".equals(propName)) { addEntriesToPatternList(pathUnmatch, propName, entry); } else if ("match_mapping_type".equals(propName)) { - matchMappingType = entry.getValue().toString(); + matchMappingType = entry.getValue(); + } else if ("unmatch_mapping_type".equals(propName)) { + unmatchMappingType = entry.getValue(); } else if ("match_pattern".equals(propName)) { matchPattern = entry.getValue().toString(); } else if ("mapping".equals(propName)) { @@ -301,29 +307,47 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe throw new MapperParsingException("template [" + name + "] must have either mapping or runtime set"); } - final XContentFieldType[] xContentFieldTypes; - if ("*".equals(matchMappingType) || (matchMappingType == null && matchPatternsAreDefined(match, pathMatch))) { - if (runtime) { - xContentFieldTypes = Arrays.stream(XContentFieldType.values()) - .filter(XContentFieldType::supportsRuntimeField) - .toArray(XContentFieldType[]::new); - } else { - xContentFieldTypes = XContentFieldType.values(); - } - } else if (matchMappingType != null) { - final XContentFieldType xContentFieldType = XContentFieldType.fromString(matchMappingType); - if (runtime && xContentFieldType.supportsRuntimeField() == false) { + if (matchMappingType == null && (unmatchMappingType != null || matchPatternsAreDefined(match, pathMatch))) { + matchMappingType = "*"; + } + XContentFieldType[] xContentFieldTypes = parseMatchMappingType(matchMappingType); + + // unmatch_mapping_type filters down the matched mapping types. + if (unmatchMappingType != null) { + final Set unmatchSet = Set.of(parseMatchMappingType(unmatchMappingType)); + Set matchSet = new LinkedHashSet(Arrays.asList(xContentFieldTypes)); + matchSet.removeAll(unmatchSet); + xContentFieldTypes = matchSet.toArray(XContentFieldType[]::new); + } + + // If match_mapping_type is "*", filter down matched mapping types + // to those allowed by runtime fields. Otherwise, throw an exception + // if match_mapping_type explicitly matches field types that are not + // allowed as runtime fields. + if (runtime && xContentFieldTypes.length > 0) { + final boolean matchAny = matchMappingType.equals("*"); + final XContentFieldType[] filteredXContentFieldTypes = Arrays.stream(xContentFieldTypes) + .filter(XContentFieldType::supportsRuntimeField) + .toArray(XContentFieldType[]::new); + final int diff = xContentFieldTypes.length - filteredXContentFieldTypes.length; + if (matchAny == false && diff > 0) { + final String[] unsupported = Arrays.stream(xContentFieldTypes) + .filter(Predicate.not(XContentFieldType::supportsRuntimeField)) + .map(XContentFieldType::toString) + .toArray(String[]::new); throw new MapperParsingException( "Dynamic template [" + name - + "] defines a runtime field but type [" - + xContentFieldType - + "] is not supported as runtime field" + + "] defines a runtime field but type" + + (diff == 1 ? "" : "s") + + " [" + + String.join(", ", unsupported) + + "] " + + (diff == 1 ? "is" : "are") + + " not supported as runtime field" ); } - xContentFieldTypes = new XContentFieldType[] { xContentFieldType }; - } else { - xContentFieldTypes = new XContentFieldType[0]; + xContentFieldTypes = filteredXContentFieldTypes; } final MatchType matchType = MatchType.fromString(matchPattern); @@ -339,6 +363,21 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xContentFieldTypes, matchType, mapping, runtime); } + private static XContentFieldType[] parseMatchMappingType(Object matchMappingType) { + final XContentFieldType[] xContentFieldTypes; + if (matchMappingType instanceof List ls) { + xContentFieldTypes = ls.stream().map(Object::toString).map(XContentFieldType::fromString).toArray(XContentFieldType[]::new); + } else if ("*".equals(matchMappingType)) { + xContentFieldTypes = XContentFieldType.values(); + } else if (matchMappingType != null) { + final XContentFieldType xContentFieldType = XContentFieldType.fromString(matchMappingType.toString()); + xContentFieldTypes = new XContentFieldType[] { xContentFieldType }; + } else { + xContentFieldTypes = new XContentFieldType[0]; + } + return xContentFieldTypes; + } + /** * @param match list of match patterns (can be empty but not null) * @param pathMatch list of pathMatch patterns (can be empty but not null) @@ -553,13 +592,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("path_unmatch", pathUnmatch); } } - // We have more than one types when (1) `match_mapping_type` is "*", and (2) match and/or path_match are defined but - // not `match_mapping_type`. In the latter the template implicitly accepts all types and we don't need to serialize - // the `match_mapping_type` values. - if (xContentFieldTypes.length > 1 && match.isEmpty() && pathMatch.isEmpty()) { - builder.field("match_mapping_type", "*"); - } else if (xContentFieldTypes.length == 1) { + // If we can match all types (considering runtime support), then we can skip serializing "match_mapping_type". + if (xContentFieldTypes.length == 1) { builder.field("match_mapping_type", xContentFieldTypes[0]); + } else if (xContentFieldTypes.length != 0) { + final long numPossibleXContentFieldTypes; + if (runtimeMapping) { + numPossibleXContentFieldTypes = List.of(XContentFieldType.values()) + .stream() + .filter(XContentFieldType::supportsRuntimeField) + .count(); + } else { + numPossibleXContentFieldTypes = XContentFieldType.values().length; + } + if (xContentFieldTypes.length < numPossibleXContentFieldTypes) { + builder.field("match_mapping_type", List.of(xContentFieldTypes).stream().map(XContentFieldType::toString).toList()); + } } if (matchType != MatchType.DEFAULT) { builder.field("match_pattern", matchType); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java index a05bf719c37af..2778aa60ffcf1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java @@ -309,6 +309,26 @@ public void testSerialization() throws Exception { assertEquals(""" {"match_mapping_type":"string","mapping":{"store":true}}""", Strings.toString(builder)); + // type-based template with single-entry array - still serializes as single string, rather than list + templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", List.of("string")); + templateDef.put("mapping", Collections.singletonMap("store", true)); + template = DynamicTemplate.parse("my_template", templateDef); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals(""" + {"match_mapping_type":"string","mapping":{"store":true}}""", Strings.toString(builder)); + + // type-based template with multi-entry array - now serializes as list + templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", List.of("string", "long")); + templateDef.put("mapping", Collections.singletonMap("store", true)); + template = DynamicTemplate.parse("my_template", templateDef); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals(""" + {"match_mapping_type":["string","long"],"mapping":{"store":true}}""", Strings.toString(builder)); + // name-based template templateDef = new HashMap<>(); if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 54db5832c2726..d3438cdaf85da 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -73,11 +73,40 @@ public void testMatchTypeOnly() throws Exception { assertThat(mapperService.fieldType("s"), notNullValue()); assertFalse(mapperService.fieldType("s").isIndexed()); - assertFalse(mapperService.fieldType("s").isSearchable()); assertThat(mapperService.fieldType("l"), notNullValue()); - assertFalse(mapperService.fieldType("s").isIndexed()); - assertTrue(mapperService.fieldType("l").isSearchable()); + assertTrue(mapperService.fieldType("l").isIndexed()); + } + + public void testUnmatchTypeOnly() throws Exception { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("unmatch_mapping_type", "string"); + b.startObject("mapping").field("index", false).endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + DocumentMapper docMapper = mapperService.documentMapper(); + ParsedDocument parsedDoc = docMapper.parse(source(b -> { + b.field("s", "hello"); + b.field("l", 1); + })); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + assertThat(mapperService.fieldType("s"), notNullValue()); + assertTrue(mapperService.fieldType("s").isIndexed()); + + assertThat(mapperService.fieldType("l"), notNullValue()); + assertFalse(mapperService.fieldType("l").isIndexed()); } public void testSimple() throws Exception { From e50dcf3d7b375ad8f0aca2ed752527b47aed374a Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Sat, 9 Dec 2023 08:23:28 +0800 Subject: [PATCH 02/11] Update server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java Co-authored-by: Felix Barnsteiner --- .../org/elasticsearch/index/mapper/DynamicTemplate.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index cd20293e40986..d4191eb1e31b5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -315,9 +315,9 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe // unmatch_mapping_type filters down the matched mapping types. if (unmatchMappingType != null) { final Set unmatchSet = Set.of(parseMatchMappingType(unmatchMappingType)); - Set matchSet = new LinkedHashSet(Arrays.asList(xContentFieldTypes)); - matchSet.removeAll(unmatchSet); - xContentFieldTypes = matchSet.toArray(XContentFieldType[]::new); + xContentFieldTypes = Stream.of(xContentFieldTypes) + .filter(Predicate.not(unmatchSet::contains)) + .toArray(XContentFieldType[]::new); } // If match_mapping_type is "*", filter down matched mapping types From e491a4d9d1b2284fbc5fc16866c6bc2ae7e50bec Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Sun, 10 Dec 2023 16:20:06 +0800 Subject: [PATCH 03/11] Address review comments - Use addEntriesToPatternList like match/path_match etc. - Serialise (un)match_mapping_type as specified, except for single-value lists which are always serialised as a non-list value. Implicit "match_mapping_type: *" is still not serialised, but an explicit setting will now be serialised where it was not before. --- .../index/mapper/DynamicTemplate.java | 151 +++++++++--------- .../mapper/DynamicTemplateParseTests.java | 58 ++++--- 2 files changed, 112 insertions(+), 97 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index d4191eb1e31b5..febb90e06d9cd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -27,6 +26,7 @@ import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import java.util.stream.Collectors; import java.util.stream.Stream; public class DynamicTemplate implements ToXContentObject { @@ -260,8 +260,8 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe List pathUnmatch = new ArrayList<>(4); Map mapping = null; boolean runtime = false; - Object matchMappingType = null; - Object unmatchMappingType = null; + List matchMappingType = new ArrayList<>(4); + List unmatchMappingType = new ArrayList<>(4); String matchPattern = MatchType.DEFAULT.toString(); for (Map.Entry entry : conf.entrySet()) { @@ -275,9 +275,9 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe } else if ("path_unmatch".equals(propName)) { addEntriesToPatternList(pathUnmatch, propName, entry); } else if ("match_mapping_type".equals(propName)) { - matchMappingType = entry.getValue(); + addEntriesToPatternList(matchMappingType, propName, entry); } else if ("unmatch_mapping_type".equals(propName)) { - unmatchMappingType = entry.getValue(); + addEntriesToPatternList(unmatchMappingType, propName, entry); } else if ("match_pattern".equals(propName)) { matchPattern = entry.getValue().toString(); } else if ("mapping".equals(propName)) { @@ -307,49 +307,49 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe throw new MapperParsingException("template [" + name + "] must have either mapping or runtime set"); } - if (matchMappingType == null && (unmatchMappingType != null || matchPatternsAreDefined(match, pathMatch))) { - matchMappingType = "*"; - } - XContentFieldType[] xContentFieldTypes = parseMatchMappingType(matchMappingType); - - // unmatch_mapping_type filters down the matched mapping types. - if (unmatchMappingType != null) { - final Set unmatchSet = Set.of(parseMatchMappingType(unmatchMappingType)); - xContentFieldTypes = Stream.of(xContentFieldTypes) - .filter(Predicate.not(unmatchSet::contains)) - .toArray(XContentFieldType[]::new); - } + // match, path_match, and unmatch_mapping_type all imply + // "match_mapping_type: *" if not explicitly specified. + final boolean wildcardMatchMappingType = ((matchMappingType.isEmpty() + && matchPatternsAreDefined(match, pathMatch, unmatchMappingType)) + || (matchMappingType.size() == 1 && matchMappingType.get(0).equals("*"))); - // If match_mapping_type is "*", filter down matched mapping types - // to those allowed by runtime fields. Otherwise, throw an exception - // if match_mapping_type explicitly matches field types that are not - // allowed as runtime fields. - if (runtime && xContentFieldTypes.length > 0) { - final boolean matchAny = matchMappingType.equals("*"); - final XContentFieldType[] filteredXContentFieldTypes = Arrays.stream(xContentFieldTypes) - .filter(XContentFieldType::supportsRuntimeField) - .toArray(XContentFieldType[]::new); - final int diff = xContentFieldTypes.length - filteredXContentFieldTypes.length; - if (matchAny == false && diff > 0) { - final String[] unsupported = Arrays.stream(xContentFieldTypes) + Stream matchXContentFieldTypes; + if (wildcardMatchMappingType) { + matchXContentFieldTypes = Stream.of(XContentFieldType.values()); + } else { + if (runtime) { + final List unsupported = matchMappingType.stream() + .map(XContentFieldType::fromString) .filter(Predicate.not(XContentFieldType::supportsRuntimeField)) .map(XContentFieldType::toString) - .toArray(String[]::new); - throw new MapperParsingException( - "Dynamic template [" - + name - + "] defines a runtime field but type" - + (diff == 1 ? "" : "s") - + " [" - + String.join(", ", unsupported) - + "] " - + (diff == 1 ? "is" : "are") - + " not supported as runtime field" - ); + .toList(); + if (unsupported.isEmpty() == false) { + final int numUnsupported = unsupported.size(); + throw new MapperParsingException( + "Dynamic template [" + + name + + "] defines a runtime field but type" + + (numUnsupported == 1 ? "" : "s") + + " [" + + String.join(", ", unsupported) + + "] " + + (numUnsupported == 1 ? "is" : "are") + + " not supported as runtime field" + ); + } } - xContentFieldTypes = filteredXContentFieldTypes; + matchXContentFieldTypes = matchMappingType.stream().map(XContentFieldType::fromString); + } + if (runtime) { + matchXContentFieldTypes = matchXContentFieldTypes.filter(XContentFieldType::supportsRuntimeField); } + final Set unmatchXContentFieldTypesSet = unmatchMappingType.stream() + .map(XContentFieldType::fromString) + .collect(Collectors.toSet()); + final XContentFieldType[] xContentFieldTypes = matchXContentFieldTypes.filter(Predicate.not(unmatchXContentFieldTypesSet::contains)) + .toArray(XContentFieldType[]::new); + final MatchType matchType = MatchType.fromString(matchPattern); List allPatterns = Stream.of(match.stream(), unmatch.stream(), pathMatch.stream(), pathUnmatch.stream()) .flatMap(s -> s) @@ -360,31 +360,27 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe matchType.validate(pattern, name); } - return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xContentFieldTypes, matchType, mapping, runtime); - } - - private static XContentFieldType[] parseMatchMappingType(Object matchMappingType) { - final XContentFieldType[] xContentFieldTypes; - if (matchMappingType instanceof List ls) { - xContentFieldTypes = ls.stream().map(Object::toString).map(XContentFieldType::fromString).toArray(XContentFieldType[]::new); - } else if ("*".equals(matchMappingType)) { - xContentFieldTypes = XContentFieldType.values(); - } else if (matchMappingType != null) { - final XContentFieldType xContentFieldType = XContentFieldType.fromString(matchMappingType.toString()); - xContentFieldTypes = new XContentFieldType[] { xContentFieldType }; - } else { - xContentFieldTypes = new XContentFieldType[0]; - } - return xContentFieldTypes; + return new DynamicTemplate( + name, + pathMatch, + pathUnmatch, + match, + unmatch, + matchMappingType, + unmatchMappingType, + xContentFieldTypes, + matchType, + mapping, + runtime + ); } /** - * @param match list of match patterns (can be empty but not null) - * @param pathMatch list of pathMatch patterns (can be empty but not null) - * @return return true if there is at least 1 match or pathMatch pattern defined + * @param matchLists zero or more lists of match patterns (can be empty but not null) + * @return return true if any of the given lists is non-empty */ - private static boolean matchPatternsAreDefined(List match, List pathMatch) { - return match.size() + pathMatch.size() > 0; + private static boolean matchPatternsAreDefined(final List... matchLists) { + return Stream.of(matchLists).anyMatch(Predicate.not(List::isEmpty)); } private static void addEntriesToPatternList(List matchList, String propName, Map.Entry entry) { @@ -411,6 +407,8 @@ private static void addEntriesToPatternList(List matchList, String propN private final List match; private final List unmatch; private final MatchType matchType; + private final List matchMappingType; + private final List unmatchMappingType; private final XContentFieldType[] xContentFieldTypes; private final Map mapping; private final boolean runtimeMapping; @@ -421,6 +419,8 @@ private DynamicTemplate( List pathUnmatch, List match, List unmatch, + List matchMappingType, + List unmatchMappingType, XContentFieldType[] xContentFieldTypes, MatchType matchType, Map mapping, @@ -432,6 +432,8 @@ private DynamicTemplate( this.match = match; this.unmatch = unmatch; this.matchType = matchType; + this.matchMappingType = matchMappingType; + this.unmatchMappingType = unmatchMappingType; this.xContentFieldTypes = xContentFieldTypes; this.mapping = mapping; this.runtimeMapping = runtimeMapping; @@ -592,21 +594,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("path_unmatch", pathUnmatch); } } - // If we can match all types (considering runtime support), then we can skip serializing "match_mapping_type". - if (xContentFieldTypes.length == 1) { - builder.field("match_mapping_type", xContentFieldTypes[0]); - } else if (xContentFieldTypes.length != 0) { - final long numPossibleXContentFieldTypes; - if (runtimeMapping) { - numPossibleXContentFieldTypes = List.of(XContentFieldType.values()) - .stream() - .filter(XContentFieldType::supportsRuntimeField) - .count(); + if (matchMappingType.isEmpty() == false) { + if (matchMappingType.size() == 1) { + builder.field("match_mapping_type", matchMappingType.get(0)); } else { - numPossibleXContentFieldTypes = XContentFieldType.values().length; + builder.field("match_mapping_type", matchMappingType); } - if (xContentFieldTypes.length < numPossibleXContentFieldTypes) { - builder.field("match_mapping_type", List.of(xContentFieldTypes).stream().map(XContentFieldType::toString).toList()); + } + if (unmatchMappingType.isEmpty() == false) { + if (unmatchMappingType.size() == 1) { + builder.field("unmatch_mapping_type", unmatchMappingType.get(0)); + } else { + builder.field("unmatch_mapping_type", unmatchMappingType); } } if (matchType != MatchType.DEFAULT) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java index 2778aa60ffcf1..a1a46e87b99ab 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java @@ -331,9 +331,6 @@ public void testSerialization() throws Exception { // name-based template templateDef = new HashMap<>(); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("match", "*name"); templateDef.put("unmatch", "first_name"); templateDef.put("mapping", Collections.singletonMap("store", true)); @@ -345,9 +342,6 @@ public void testSerialization() throws Exception { // name-based template with array of match patterns templateDef = new HashMap<>(); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("match", List.of("*name", "user*")); templateDef.put("unmatch", "first_name"); templateDef.put("mapping", Collections.singletonMap("store", true)); @@ -357,13 +351,22 @@ public void testSerialization() throws Exception { assertEquals(""" {"match":["*name","user*"],"unmatch":"first_name","mapping":{"store":true}}""", Strings.toString(builder)); + // name-based template with explicit match_mapping_type wildcard pattern + templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "*"); + templateDef.put("match", "*name"); + templateDef.put("unmatch", "first_name"); + templateDef.put("mapping", Collections.singletonMap("store", true)); + template = DynamicTemplate.parse("my_template", templateDef); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals(""" + {"match":"*name","unmatch":"first_name","match_mapping_type":"*","mapping":{"store":true}}""", Strings.toString(builder)); + // path-based template templateDef = new HashMap<>(); templateDef.put("path_match", "*name"); templateDef.put("path_unmatch", "first_name"); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("mapping", Collections.singletonMap("store", true)); template = DynamicTemplate.parse("my_template", templateDef); builder = JsonXContent.contentBuilder(); @@ -375,9 +378,6 @@ public void testSerialization() throws Exception { templateDef = new HashMap<>(); templateDef.put("path_match", List.of("*name")); templateDef.put("path_unmatch", List.of("first_name")); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("mapping", Collections.singletonMap("store", true)); template = DynamicTemplate.parse("my_template", templateDef); builder = JsonXContent.contentBuilder(); @@ -389,9 +389,6 @@ public void testSerialization() throws Exception { templateDef = new HashMap<>(); templateDef.put("path_match", List.of("*name", "user*")); templateDef.put("path_unmatch", List.of("first_name", "username")); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("mapping", Collections.singletonMap("store", true)); template = DynamicTemplate.parse("my_template", templateDef); builder = JsonXContent.contentBuilder(); @@ -406,9 +403,6 @@ public void testSerialization() throws Exception { templateDef = new HashMap<>(); templateDef.put("match", "^a$"); templateDef.put("match_pattern", "regex"); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("mapping", Collections.singletonMap("store", true)); template = DynamicTemplate.parse("my_template", templateDef); builder = JsonXContent.contentBuilder(); @@ -424,6 +418,31 @@ public void testSerialization() throws Exception { template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertThat(Strings.toString(builder), equalTo(""" {"mapping":{"store":true}}""")); + + // match_mapping_type and unmatch_mapping_type with single values + templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", "*"); + templateDef.put("unmatch_mapping_type", "string"); + templateDef.put("mapping", Collections.singletonMap("store", true)); + template = DynamicTemplate.parse("my_template", templateDef); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals(""" + {"match_mapping_type":"*","unmatch_mapping_type":"string","mapping":{"store":true}}""", Strings.toString(builder)); + + // match_mapping_type and unmatch_mapping_type with multi-entry arrays + templateDef = new HashMap<>(); + templateDef.put("match_mapping_type", List.of("string", "object")); + templateDef.put("unmatch_mapping_type", List.of("object", "string")); + templateDef.put("mapping", Collections.singletonMap("store", true)); + template = DynamicTemplate.parse("my_template", templateDef); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals( + """ + {"match_mapping_type":["string","object"],"unmatch_mapping_type":["object","string"],"mapping":{"store":true}}""", + Strings.toString(builder) + ); } public void testSerializationRuntimeMappings() throws Exception { @@ -441,9 +460,6 @@ public void testSerializationRuntimeMappings() throws Exception { templateDef = new HashMap<>(); templateDef.put("match", "*name"); templateDef.put("unmatch", "first_name"); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("runtime", Collections.singletonMap("type", "new_type")); template = DynamicTemplate.parse("my_template", templateDef); builder = JsonXContent.contentBuilder(); From ccfda067a922ea983359ed2ad15f09ad521a2217 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 11 Dec 2023 08:19:42 +0800 Subject: [PATCH 04/11] Fix test --- .../index/mapper/DynamicTemplateParseTests.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java index a1a46e87b99ab..4df45f04484ac 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java @@ -471,9 +471,6 @@ public void testSerializationRuntimeMappings() throws Exception { templateDef = new HashMap<>(); templateDef.put("path_match", "*name"); templateDef.put("path_unmatch", "first_name"); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("runtime", Collections.emptyMap()); template = DynamicTemplate.parse("my_template", templateDef); builder = JsonXContent.contentBuilder(); @@ -484,9 +481,6 @@ public void testSerializationRuntimeMappings() throws Exception { // regex matching templateDef = new HashMap<>(); templateDef.put("match", "^a$"); - if (randomBoolean()) { - templateDef.put("match_mapping_type", "*"); - } templateDef.put("match_pattern", "regex"); templateDef.put("runtime", Collections.emptyMap()); template = DynamicTemplate.parse("my_template", templateDef); From b6c81341690c9c88ccceb7a205e61663d70be574 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 11 Dec 2023 09:46:42 +0800 Subject: [PATCH 05/11] Add YAML REST test --- .../indices.create/30_dynamic_template.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml index 648ebb1705123..15b49e8d2fbec 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml @@ -59,3 +59,32 @@ - match: { test_index.mappings.dynamic_templates.0.mytemplate.path_match.1: "user.name.*"} - match: { test_index.mappings.dynamic_templates.0.mytemplate.path_unmatch: "*.middle"} - match: { test_index.mappings.dynamic_templates.0.mytemplate.mapping.type: "keyword" } + +--- +"Create index with dynamic_mappings, with wildcard match_mapping_type and an unmatch_mapping_type array": + - skip: + version: " - 8.12.99" + reason: unmatch_mapping_type in dynamic templates added in 8.13 + - do: + indices.create: + index: test_index + body: + mappings: + dynamic_templates: + - mytemplate: + match_mapping_type: "*" + unmatch_mapping_type: + - "object" + - "boolean" + mapping: + type: long + + - do: + indices.get_mapping: + index: test_index + + - is_true: test_index.mappings + - match: { test_index.mappings.dynamic_templates.0.mytemplate.match_mapping_type: "*"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.unmatch_mapping_type.0: "object"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.unmatch_mapping_type.1: "boolean"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.mapping.type: "long" } From 952d8167cade5164f42a01b9779ddc2c3a973622 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 11 Dec 2023 10:43:37 +0800 Subject: [PATCH 06/11] Add another unit test for unmatch_mapping_type ... to highlight the original purpose, which is to unmatch an internal object field whose path may match a leaf field. --- .../mapper/DynamicTemplateParseTests.java | 9 +++++++ .../index/mapper/DynamicTemplatesTests.java | 26 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java index 4df45f04484ac..35f779f97b024 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java @@ -298,6 +298,15 @@ public void testSupportedMatchMappingTypesRuntime() { } } + public void testUnmatchMappingType() { + Map templateDef = new HashMap<>(); + templateDef.put("unmatch_mapping_type", "object"); + templateDef.put("mapping", Collections.singletonMap("store", true)); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef); + assertTrue(template.match(null, "a.b", "b", XContentFieldType.STRING)); + assertFalse(template.match(null, "a.b", "b", XContentFieldType.OBJECT)); + } + public void testSerialization() throws Exception { // type-based template Map templateDef = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index d3438cdaf85da..afb791c6d4194 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -2541,4 +2541,30 @@ record MatchPattern(String pattern, boolean isValid) {} } } } + + public void testUnmatchTypeWithPathMatch() throws Exception { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + // unmatch_mapping_type prevents the first "b" in "a.b.b" from matching. + b.field("unmatch_mapping_type", "object"); + b.field("path_match", "*.b"); + b.startObject("mapping").field("type", "double").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + DocumentMapper docMapper = mapperService.documentMapper(); + ParsedDocument parsedDoc = docMapper.parse(source(b -> { b.field("a.b.b", 123.456); })); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + assertEquals("double", mapperService.fieldType("a.b.b").typeName()); + } } From 10276ed056adcac8ffe9db3fa8604c9874cfa481 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 11 Dec 2023 15:28:08 +0800 Subject: [PATCH 07/11] Update docs/changelog/103171.yaml --- docs/changelog/103171.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/changelog/103171.yaml diff --git a/docs/changelog/103171.yaml b/docs/changelog/103171.yaml new file mode 100644 index 0000000000000..95ad6a1ea77c2 --- /dev/null +++ b/docs/changelog/103171.yaml @@ -0,0 +1,7 @@ +pr: 103171 +summary: "Add `unmatch_mapping_type`, and support array of types" +area: Mapping +type: feature +issues: + - 102807 + - 102795 From 78a000714c95b970f487c9b2d78219b64c80975a Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 12 Dec 2023 13:43:30 +0800 Subject: [PATCH 08/11] Address review comment Move runtime-supported type filtering. --- .../org/elasticsearch/index/mapper/DynamicTemplate.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index febb90e06d9cd..de08b6979463c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -316,6 +316,9 @@ && matchPatternsAreDefined(match, pathMatch, unmatchMappingType)) Stream matchXContentFieldTypes; if (wildcardMatchMappingType) { matchXContentFieldTypes = Stream.of(XContentFieldType.values()); + if (runtime) { + matchXContentFieldTypes = matchXContentFieldTypes.filter(XContentFieldType::supportsRuntimeField); + } } else { if (runtime) { final List unsupported = matchMappingType.stream() @@ -340,9 +343,6 @@ && matchPatternsAreDefined(match, pathMatch, unmatchMappingType)) } matchXContentFieldTypes = matchMappingType.stream().map(XContentFieldType::fromString); } - if (runtime) { - matchXContentFieldTypes = matchXContentFieldTypes.filter(XContentFieldType::supportsRuntimeField); - } final Set unmatchXContentFieldTypesSet = unmatchMappingType.stream() .map(XContentFieldType::fromString) From cfdf7e8caf2e06582e2049721dd1d68fce66ead0 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 12 Dec 2023 15:58:42 +0800 Subject: [PATCH 09/11] Update docs --- .../mapping/dynamic/templates.asciidoc | 43 ++++++++++++++++--- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index 6f2cec356edb4..88ebf63c826be 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -7,8 +7,8 @@ dynamic mapping by setting the dynamic parameter to `true` or `runtime`. You can then use dynamic templates to define custom mappings that can be applied to dynamically added fields based on the matching condition: -* <> operates on the data type that -{es} detects +* <> +operate on the data type that {es} detects * <> use a pattern to match on the field name * <> operate on the full @@ -116,10 +116,13 @@ See <> for how to use dynamic templates to map `string` fields as either indexed fields or runtime fields. [[match-mapping-type]] -==== `match_mapping_type` +==== `match_mapping_type` and `unmatch_mapping_type` -The `match_mapping_type` is the data type detected by the JSON parser. Because -JSON doesn't distinguish a `long` from an `integer` or a `double` from +The `match_mapping_type` parameter matches fields by the data type detected by +the JSON parser, while `unmatch_mapping_type` excludes fields based on the data +type. + +Because JSON doesn't distinguish a `long` from an `integer` or a `double` from a `float`, any parsed floating point number is considered a `double` JSON data type, while any parsed `integer` number is considered a `long`. @@ -132,7 +135,10 @@ which is why `"dynamic":"runtime"` uses `double`. include::field-mapping.asciidoc[tag=dynamic-field-mapping-types-tag] -Use a wildcard (`*`) to match all data types. +You can specify either a single data type or a list of data types for either +the `match_mapping_type` or `unmatch_mapping_type` parameters. You can also +use a wildcard (`*`) for the `match_mapping_type` parameter to match all +data types. For example, if we wanted to map all integer fields as `integer` instead of `long`, and all `string` fields as both `text` and `keyword`, we @@ -144,6 +150,16 @@ PUT my-index-000001 { "mappings": { "dynamic_templates": [ + { + "numeric_counts": { + "match_mapping_type": ["long", "double"], + "match": "count", + "mapping": { + "type": "{dynamic_type}" + "index": false + } + } + }, { "integers": { "match_mapping_type": "long", @@ -165,6 +181,15 @@ PUT my-index-000001 } } } + }, + { + "non_objects_keyword": { + "match_mapping_type": "*", + "unmatch_mapping_type": "object", + "mapping": { + "type": "keyword" + } + } } ] } @@ -173,12 +198,16 @@ PUT my-index-000001 PUT my-index-000001/_doc/1 { "my_integer": 5, <1> - "my_string": "Some string" <2> + "my_string": "Some string", <2> + "my_boolean": "false", <3> + "field": {"count": 4} <4> } -------------------------------------------------- <1> The `my_integer` field is mapped as an `integer`. <2> The `my_string` field is mapped as a `text`, with a `keyword` <>. +<3> The `my_boolean` field is mapped as a `keyword`. +<4> The `field.count` field is mapped as a `long`. [[match-unmatch]] ==== `match` and `unmatch` From 8aef59f230de84ca63260e8888e109ad223a7e21 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 12 Dec 2023 16:34:55 +0800 Subject: [PATCH 10/11] Fix docs --- docs/reference/mapping/dynamic/templates.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index 88ebf63c826be..af89c265db2ca 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -155,7 +155,7 @@ PUT my-index-000001 "match_mapping_type": ["long", "double"], "match": "count", "mapping": { - "type": "{dynamic_type}" + "type": "{dynamic_type}", "index": false } } From 2592cbee9add1739416d4398a518f661300901f4 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 9 Feb 2024 15:40:01 +0100 Subject: [PATCH 11/11] Extract method to serialize a string or array field, depending on the size of the list --- .../index/mapper/DynamicTemplate.java | 58 +++++-------------- 1 file changed, 16 insertions(+), 42 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index de08b6979463c..b9230c835cb59 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -566,48 +566,12 @@ Map getMapping() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (match.isEmpty() == false) { - if (match.size() == 1) { - builder.field("match", match.get(0)); - } else { - builder.field("match", match); - } - } - if (pathMatch.isEmpty() == false) { - if (pathMatch.size() == 1) { - builder.field("path_match", pathMatch.get(0)); - } else { - builder.field("path_match", pathMatch); - } - } - if (unmatch.isEmpty() == false) { - if (unmatch.size() == 1) { - builder.field("unmatch", unmatch.get(0)); - } else { - builder.field("unmatch", unmatch); - } - } - if (pathUnmatch.isEmpty() == false) { - if (pathUnmatch.size() == 1) { - builder.field("path_unmatch", pathUnmatch.get(0)); - } else { - builder.field("path_unmatch", pathUnmatch); - } - } - if (matchMappingType.isEmpty() == false) { - if (matchMappingType.size() == 1) { - builder.field("match_mapping_type", matchMappingType.get(0)); - } else { - builder.field("match_mapping_type", matchMappingType); - } - } - if (unmatchMappingType.isEmpty() == false) { - if (unmatchMappingType.size() == 1) { - builder.field("unmatch_mapping_type", unmatchMappingType.get(0)); - } else { - builder.field("unmatch_mapping_type", unmatchMappingType); - } - } + addStringOrArrayField(builder, "match", match); + addStringOrArrayField(builder, "path_match", pathMatch); + addStringOrArrayField(builder, "unmatch", unmatch); + addStringOrArrayField(builder, "path_unmatch", pathUnmatch); + addStringOrArrayField(builder, "match_mapping_type", matchMappingType); + addStringOrArrayField(builder, "unmatch_mapping_type", unmatchMappingType); if (matchType != MatchType.DEFAULT) { builder.field("match_pattern", matchType); } @@ -620,4 +584,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + private void addStringOrArrayField(XContentBuilder builder, String fieldName, List list) throws IOException { + if (list.isEmpty() == false) { + if (list.size() == 1) { + builder.field(fieldName, list.get(0)); + } else { + builder.field(fieldName, list); + } + } + } }