diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 889dc1fe19a3c..65123c06b7649 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -83,7 +83,7 @@ static class TokenCountFieldType extends NumberFieldMapper.NumberFieldType { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { if (hasDocValues() == false) { - return lookup -> org.elasticsearch.common.collect.List.of(); + return (lookup, ignoredFields) -> org.elasticsearch.common.collect.List.of(); } return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml index e849d635a3e91..18b2808e18a4e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -402,3 +402,407 @@ Test unmapped fields inside disabled objects: - match: hits.hits.0.fields.f1\.a: - b +--- +Test nested fields: + - skip: + version: ' - 7.99.99' + reason: support was introduced in 8.0.0 and not backported yet + - do: + indices.create: + index: test + body: + mappings: + properties: + products: + type: nested + properties: + manufacturer: + type: keyword + base_price: + type: double + product_id: + type: integer + + - do: + index: + index: test + id: 1 + refresh: true + body: + products: + - { "manufacturer" : "Supersoft", "base_price" : 1.55, "product_id" : 12345} + - { "manufacturer" : "HyperSmart", "base_price" : 20.20, "product_id" : 54321} + + - do: + search: + index: test + body: + _source: false + fields: [ "*" ] + - length: { hits.hits.0.fields : 1 } + - match: + hits.hits.0.fields.products.0: { "manufacturer" : ["Supersoft"], "base_price" : [1.55], "product_id" : [12345]} + - match: + hits.hits.0.fields.products.1: { "manufacturer" : ["HyperSmart"], "base_price" : [20.20], "product_id" : [54321]} + + - do: + search: + index: test + body: + _source: false + fields: ["products.manufacturer", "products.base_price"] + - length: { hits.hits.0.fields : 1 } + - match: + hits.hits.0.fields.products.0: { "manufacturer" : ["Supersoft"], "base_price" : [1.55]} + - match: + hits.hits.0.fields.products.1: { "manufacturer" : ["HyperSmart"], "base_price" : [20.20]} + + - do: + search: + index: test + body: + _source: false + fields: ["products.manufacturer"] + - length: { hits.hits.0.fields : 1 } + - match: + hits.hits.0.fields.products.0: { "manufacturer" : ["Supersoft"]} + - match: + hits.hits.0.fields.products.1: { "manufacturer" : ["HyperSmart"]} + + - do: + search: + index: test + body: + _source: false + fields: ["products"] + - is_false: hits.hits.0.fields +--- +Test nested field inside object structure: + - skip: + version: ' - 7.99.99' + reason: support was introduced in 8.0.0 and not backported yet + - do: + indices.create: + index: test + body: + mappings: + properties: + obj: + type: object + properties: + products: + type: nested + properties: + manufacturer: + type: keyword + base_price: + type: double + product_id: + type: integer + other_obj_field: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + obj: + products: + - { "manufacturer" : "Supersoft", "base_price" : 1.55, "product_id" : 12345} + - { "manufacturer" : "HyperSmart", "base_price" : 20.20, "product_id" : 54321} + other_obj_field: other_value + - do: + index: + index: test + id: 2 + refresh: true + body: + obj.products: + - { "manufacturer" : "RealTec", "base_price" : 12.55, "product_id" : 23451} + obj.other_obj_field: other_value2 + + + - do: + search: + index: test + body: + _source: false + fields: [ "*" ] + - length: { hits.hits.0.fields : 2 } + - match: + hits.hits.0.fields.obj\.products.0: { "manufacturer" : ["Supersoft"], "base_price" : [1.55], "product_id" : [12345]} + - match: + hits.hits.0.fields.obj\.products.1: { "manufacturer" : ["HyperSmart"], "base_price" : [20.20], "product_id" : [54321]} + - match: + hits.hits.0.fields.obj\.other_obj_field.0: other_value + - match: + hits.hits.1.fields.obj\.products.0: { "manufacturer" : ["RealTec"], "base_price" : [12.55], "product_id" : [23451]} + - match: + hits.hits.1.fields.obj\.other_obj_field.0: other_value2 + + - do: + search: + index: test + body: + _source: false + fields: ["obj.other_obj_field"] + - match: + hits.hits.0.fields.obj\.other_obj_field.0: other_value + + - do: + search: + index: test + body: + _source: false + fields: ["obj.products.manufacturer"] + - match: + hits.hits.0.fields.obj\.products.0: { "manufacturer" : ["Supersoft"]} + - match: + hits.hits.0.fields.obj\.products.1: { "manufacturer" : ["HyperSmart"]} + - match: + hits.hits.1.fields.obj\.products.0: { "manufacturer" : ["RealTec"]} + + - do: + search: + index: test + body: + _source: false + fields: ["obj.pro*ts.manufacturer"] + - match: + hits.hits.0.fields.obj\.products.0: { "manufacturer" : ["Supersoft"]} + - match: + hits.hits.0.fields.obj\.products.1: { "manufacturer" : ["HyperSmart"]} + - match: + hits.hits.1.fields.obj\.products.0: { "manufacturer" : ["RealTec"]} +--- +Test doubly nested fields: + - skip: + version: ' - 7.99.99' + reason: support was introduced in 8.0.0 and not backported yet + - do: + indices.create: + index: test + body: + mappings: + properties: + id: + type: keyword + user: + type: nested + properties: + first: + type: keyword + last: + type: keyword + fields: + keyword: + type : keyword + address: + type: nested + - do: + index: + index: test + id: 1 + refresh: true + body: + id: abcd1234 + user: + - { "first" : "John", "address" : { "city" : "Berlin" }, "account" : { "size" : 1213 }} + - { "first" : "Alice", "last" : "White", "address" : [{ "city" : "Toronto", "zip" : "1111" },{ "city" : "Ottawa", "zip" : "2222" }]} + - { "first" : "John", "last" : "Snow" } + + - do: + search: + index: test + body: + _source: false + fields: [ { "field" : "*" } ] + - match: + hits.hits.0.fields.id: + - abcd1234 + - is_false: hits.hits.0.fields.user\.first + - is_false: hits.hits.0.fields.user\.last + - is_false: hits.hits.0.fields.user\.account\.size + - match: + hits.hits.0.fields.user.0: { "first" : [ "John" ], "address" : [{ "city" : ["Berlin"], "city.keyword" : ["Berlin"]}], "account.size" : [1213] } + - match: + hits.hits.0.fields.user.1: { "first" : [ "Alice" ], "last" : [ "White" ], "last.keyword" : [ "White" ], "address" : [ { "zip" : [ "1111" ], "zip.keyword" : [ "1111" ], "city" : [ "Toronto" ], "city.keyword" : [ "Toronto" ] }, { "zip" : ["2222"], "zip.keyword" : ["2222"], "city" : ["Ottawa"], "city.keyword" : [ "Ottawa" ]}] } + - match: + hits.hits.0.fields.user.2: { "first" : [ "John" ], "last" : [ "Snow" ], "last.keyword" : [ "Snow" ] } + + - do: + search: + index: test + body: + _source: false + fields: [ { "field" : "user.address*" } ] + + - match: + hits.hits.0.fields.user.0: { "address" : [{ "city" : ["Berlin"], "city.keyword" : ["Berlin"]}] } + - match: + hits.hits.0.fields.user.1: { "address" : [ { "zip" : [ "1111" ], "zip.keyword" : [ "1111" ], "city" : [ "Toronto" ], "city.keyword" : [ "Toronto" ] }, { "zip" : ["2222"], "zip.keyword" : ["2222"], "city" : ["Ottawa"], "city.keyword" : [ "Ottawa" ]}] } + - length: { hits.hits.0.fields.user : 2 } + +--- +Test nested fields with unmapped subfields: + - skip: + version: ' - 7.99.99' + reason: support was introduced in 8.0.0 and not backported yet + - do: + indices.create: + index: test + body: + mappings: + properties: + id: + type: keyword + user: + type: nested + properties: + first: + type: keyword + address: + type: object + enabled: false + user_account: + type: nested + properties: + details: + type: object + enabled: false + + - do: + index: + index: test + id: 1 + refresh: true + body: + id: abcd1234 + user_account: + - { "details" : { "id" : "xyz1234" }} + user: + - { "first" : "John", "address" : { "city" : "Berlin" }} + + - do: + search: + index: test + body: + _source: false + fields: + - { "field" : "*", "include_unmapped" : true } + - match: + hits.hits.0.fields.id: + - abcd1234 + - is_false: hits.hits.0.fields.user\.first + - is_false: hits.hits.0.fields.user\.last + - is_false: hits.hits.0.fields.user_account\.details\.id + - match: + hits.hits.0.fields.user.0: { "first" : [ "John" ], "address.city" : ["Berlin"]} + - match: + hits.hits.0.fields.user_account.0: { "details.id" : ["xyz1234"]} + + - do: + search: + index: test + body: + _source: false + fields: + - { "field" : "user.address.*", "include_unmapped" : true } + - match: + hits.hits.0.fields.user.0: { "address.city" : ["Berlin"]} +--- +Test nested fields with ignored subfields: + - skip: + version: ' - 7.99.99' + reason: support was introduced in 8.0.0 and not backported yet + - do: + indices.create: + index: test + body: + mappings: + properties: + malformed_outside: + type: integer + ignore_malformed: true + user: + type: nested + properties: + malformed_inside: + type: integer + ignore_malformed: true + first: + type: keyword + - do: + index: + index: test + id: 1 + refresh: true + body: + malformed_outside : "bad_value_1" + user: + - { "first" : "John", "malformed_inside" : "bad_value_2"} + + - do: + search: + index: test + body: + _source: false + fields: + - { "field" : "*", "include_unmapped" : true } + - is_false: hits.hits.0.fields.malformed_outside + - match: + hits.hits.0.fields.user: + - { "first" : [ "John" ] } +--- +Test nested field with sibling field resolving to DocValueFetcher: + - skip: + version: ' - 7.99.99' + reason: support was introduced in 8.0.0 and not backported yet + - do: + indices.create: + index: test + body: + mappings: + properties: + owner: + type: text + fields: + length: + type: token_count + analyzer: standard + products: + type: nested + properties: + manufacturer: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + owner: "Anna Ott" + products: + - { "manufacturer" : "Supersoft"} + - { "manufacturer" : "HyperSmart"} + + - do: + search: + index: test + body: + _source: false + fields: [ "*" ] + - length: { hits.hits.0.fields : 3 } + - match: + hits.hits.0.fields.owner: + - "Anna Ott" + - match: + hits.hits.0.fields.owner\.length: + - 2 + - match: + hits.hits.0.fields.products.0: { "manufacturer" : ["Supersoft"]} + - match: + hits.hits.0.fields.products.1: { "manufacturer" : ["HyperSmart"]} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index 2119ada804594..995436af86103 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -165,7 +165,10 @@ public void testSimpleNested() throws Exception { assertThat(innerHits.getAt(0).getHighlightFields().get("comments.message").getFragments()[0].string(), equalTo("fox eat quick")); assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(comments.message:fox in")); - assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("fox eat quick")); + assertThat( + innerHits.getAt(0).getFields().get("comments").getValue(), + equalTo(Collections.singletonMap("message", Collections.singletonList("fox eat quick"))) + ); assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5")); response = client().prepareSearch("articles") diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index 8da02b9a7c111..6d004a12a54c4 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -110,6 +111,31 @@ public static Object extractValue(Map map, String... pathElements) { return XContentMapValues.extractValue(pathElements, 0, map, null); } + /** + * For the provided nested path, return its value in the xContent map. + * + * @param nestedPath the nested field value's path in the map. + * + * @return the list associated with the path in the map or {@code null} if the path does not exits. + */ + public static List extractNestedValue(String nestedPath, Map map) { + Object extractedValue = XContentMapValues.extractValue(nestedPath, map); + List nestedParsedSource = null; + if (extractedValue != null) { + if (extractedValue instanceof List) { + // nested field has an array value in the _source + nestedParsedSource = (List) extractedValue; + } else if (extractedValue instanceof Map) { + // nested field has an object value in the _source. This just means the nested field has just one inner object, + // which is valid, but uncommon. + nestedParsedSource = Collections.singletonList(extractedValue); + } else { + throw new IllegalStateException("extracted source isn't an object or an array"); + } + } + return nestedParsedSource; + } + /** * For the provided path, return its value in the xContent map. * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index 1bdca1f89669b..d213b3bc44131 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -26,8 +26,8 @@ */ public abstract class ArraySourceValueFetcher implements ValueFetcher { private final Set sourcePaths; - private final @Nullable - Object nullValue; + private final @Nullable Object nullValue; + private final String fieldName; public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -41,11 +41,15 @@ public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { this.sourcePaths = context.sourcePath(fieldName); this.nullValue = nullValue; + this.fieldName = fieldName; } @Override - public List fetchValues(SourceLookup lookup) { + public List fetchValues(SourceLookup lookup, Set ignoredFields) { List values = new ArrayList<>(); + if (ignoredFields.contains(fieldName)) { + return values; + } for (String path : sourcePaths) { Object sourceValue = lookup.extractValue(path, nullValue); if (sourceValue == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java index c28571f3ecb39..30851e01037b8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Set; import static java.util.Collections.emptyList; @@ -32,12 +33,13 @@ public DocValueFetcher(DocValueFormat format, IndexFieldData ifd) { this.ifd = ifd; } + @Override public void setNextReader(LeafReaderContext context) { leaf = ifd.load(context).getLeafValueFetcher(format); } @Override - public List fetchValues(SourceLookup lookup) throws IOException { + public List fetchValues(SourceLookup lookup, Set ignoredFields) throws IOException { if (false == leaf.advanceExact(lookup.docId())) { return emptyList(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java new file mode 100644 index 0000000000000..05e16d2681278 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.search.fetch.subphase.FieldFetcher; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class NestedValueFetcher implements ValueFetcher { + + private final String nestedFieldPath; + private final FieldFetcher nestedFieldFetcher; + + // the name of the nested field without the full path, i.e. in foo.bar.baz it would be baz + private final String nestedFieldName; + private final String[] nestedPathParts; + + public NestedValueFetcher(String nestedField, FieldFetcher nestedFieldFetcher) { + assert nestedField != null && nestedField.isEmpty() == false; + this.nestedFieldPath = nestedField; + this.nestedFieldFetcher = nestedFieldFetcher; + this.nestedPathParts = nestedFieldPath.split("\\."); + this.nestedFieldName = nestedPathParts[nestedPathParts.length - 1]; + } + + @Override + public List fetchValues(SourceLookup lookup, Set ignoreFields) throws IOException { + List nestedEntriesToReturn = new ArrayList<>(); + Map filteredSource = new HashMap<>(); + Map stub = createSourceMapStub(filteredSource); + List nestedValues = XContentMapValues.extractNestedValue(nestedFieldPath, lookup.source()); + if (nestedValues == null) { + return Collections.emptyList(); + } + for (Object entry : nestedValues) { + // add this one entry only to the stub and use this as source lookup + stub.put(nestedFieldName, entry); + SourceLookup nestedSourceLookup = new SourceLookup(); + nestedSourceLookup.setSource(filteredSource); + + Map fetchResult = nestedFieldFetcher.fetch(nestedSourceLookup, ignoreFields); + + Map nestedEntry = new HashMap<>(); + for (DocumentField field : fetchResult.values()) { + List fetchValues = field.getValues(); + if (fetchValues.isEmpty() == false) { + String keyInNestedMap = field.getName().substring(nestedFieldPath.length() + 1); + nestedEntry.put(keyInNestedMap, fetchValues); + } + } + if (nestedEntry.isEmpty() == false) { + nestedEntriesToReturn.add(nestedEntry); + } + } + return nestedEntriesToReturn; + } + + // create a filtered source map stub which contains the nested field path + private Map createSourceMapStub(Map filteredSource) { + Map next = filteredSource; + for (int i = 0; i < nestedPathParts.length - 1; i++) { + String part = nestedPathParts[i]; + Map newMap = new HashMap<>(); + next.put(part, newMap); + next = newMap; + } + return next; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 2fc183e9390e5..564e74ebddd2c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -27,8 +27,8 @@ */ public abstract class SourceValueFetcher implements ValueFetcher { private final Set sourcePaths; - private final @Nullable - Object nullValue; + private final @Nullable Object nullValue; + private final String fieldName; public SourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -42,11 +42,15 @@ public SourceValueFetcher(String fieldName, SearchExecutionContext context) { public SourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { this.sourcePaths = context.sourcePath(fieldName); this.nullValue = nullValue; + this.fieldName = fieldName; } @Override - public List fetchValues(SourceLookup lookup) { + public List fetchValues(SourceLookup lookup, Set ignoredFields) { List values = new ArrayList<>(); + if (ignoredFields.contains(fieldName)) { + return values; + } for (String path : sourcePaths) { Object sourceValue = lookup.extractValue(path, nullValue); if (sourceValue == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java index 8cf0e8fe6fdbf..83d5ce6682d55 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java @@ -9,11 +9,13 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.Nullable; import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.List; +import java.util.Set; /** * A helper class for fetching field values during the {@link FetchFieldsPhase}. Each {@link MappedFieldType} @@ -21,19 +23,20 @@ */ public interface ValueFetcher { /** - * Given access to a document's _source, return this field's values. - * - * In addition to pulling out the values, they will be parsed into a standard form. - * For example numeric field mappers make sure to parse the source value into a number - * of the right type. - * - * Note that for array values, the order in which values are returned is undefined and - * should not be relied on. - * - * @param lookup a lookup structure over the document's source. - * @return a list a standardized field values. - */ - List fetchValues(SourceLookup lookup) throws IOException; + * Given access to a document's _source, return this field's values. + * + * In addition to pulling out the values, they will be parsed into a standard form. + * For example numeric field mappers make sure to parse the source value into a number + * of the right type. + * + * Note that for array values, the order in which values are returned is undefined and + * should not be relied on. + * + * @param lookup a lookup structure over the document's source. + * @param ignoredFields the fields in _ignored that have been ignored for this document because they were malformed + * @return a list a standardized field values. + */ + List fetchValues(SourceLookup lookup, @Nullable Set ignoredFields) throws IOException; /** * Update the leaf reader used to fetch values. diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index cc715c67a799d..8607d56b82cf6 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -303,6 +303,10 @@ public boolean hasMappings() { return mappingLookup.hasMappings(); } + public List nestedMappings() { + return mappingLookup.getNestedMappers(); + } + /** * Returns all the fields that match a given pattern. If prefixed with a * type then the fields will be returned with a type prefix. @@ -659,6 +663,17 @@ public MappingLookup.CacheKey mappingCacheKey() { return mappingLookup.cacheKey(); } + /** + * Given a nested object path, returns the path to its nested parent + * + * In particular, if a nested field `foo` contains an object field + * `bar.baz`, then calling this method with `foo.bar.baz` will return + * the path `foo`, skipping over the object-but-not-nested `foo.bar` + */ + public String getNestedParent(String nestedPath) { + return mappingLookup.getNestedParent(nestedPath); + } + public NestedDocuments getNestedDocuments() { return new NestedDocuments(mappingLookup, indexVersionCreated(), bitsetFilterCache::getBitSetProducer); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 49cf658b9d273..e92313b8f48d8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -390,18 +390,7 @@ private HitContext prepareNestedHitContext(SearchContext context, for (SearchHit.NestedIdentity nested = nestedIdentity; nested != null; nested = nested.getChild()) { String nestedPath = nested.getField().string(); current.put(nestedPath, new HashMap<>()); - Object extractedValue = XContentMapValues.extractValue(nestedPath, rootSourceAsMap); - List nestedParsedSource; - if (extractedValue instanceof List) { - // nested field has an array value in the _source - nestedParsedSource = (List) extractedValue; - } else if (extractedValue instanceof Map) { - // nested field has an object value in the _source. This just means the nested field has just one inner object, - // which is valid, but uncommon. - nestedParsedSource = Collections.singletonList(extractedValue); - } else { - throw new IllegalStateException("extracted source isn't an object or an array"); - } + List nestedParsedSource = XContentMapValues.extractNestedValue(nestedPath, rootSourceAsMap); if ((nestedParsedSource.get(0) instanceof Map) == false && hasNonNestedParent.test(nestedPath)) { // When one of the parent objects are not nested then XContentMapValues.extractValue(...) extracts the values // from two or more layers resulting in a list of list being returned. This is because nestedPath diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index ef10bfc4ac6d0..837f9eb5c45c9 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -84,7 +85,7 @@ public void process(HitContext hit) throws IOException { // docValues fields will still be document fields, and put under "fields" section of a hit. hit.hit().setDocumentField(f.field, hitField); } - hitField.getValues().addAll(f.fetcher.fetchValues(hit.sourceLookup())); + hitField.getValues().addAll(f.fetcher.fetchValues(hit.sourceLookup(), Collections.emptySet())); } } }; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index e9c1a55ab163f..b734aceabcc30 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -15,7 +15,6 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; @@ -36,15 +35,12 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) { return null; } - SearchLookup searchLookup = fetchContext.searchLookup(); if (fetchContext.getSearchExecutionContext().isSourceEnabled() == false) { throw new IllegalArgumentException("Unable to retrieve the requested [fields] since _source is disabled " + "in the mappings for index [" + fetchContext.getIndexName() + "]"); } - FieldFetcher fieldFetcher = FieldFetcher.create(fetchContext.getSearchExecutionContext(), - searchLookup, - fetchFieldsContext.fields()); + FieldFetcher fieldFetcher = FieldFetcher.create(fetchContext.getSearchExecutionContext(), fetchFieldsContext.fields()); return new FetchSubPhaseProcessor() { @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java index 54155a6f476b4..e44c2a20c74cb 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java @@ -9,47 +9,59 @@ package org.elasticsearch.search.fetch.subphase; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NestedValueFetcher; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * A helper class to {@link FetchFieldsPhase} that's initialized with a list of field patterns to fetch. * Then given a specific document, it can retrieve the corresponding fields from the document's source. */ public class FieldFetcher { + public static FieldFetcher create(SearchExecutionContext context, - SearchLookup searchLookup, - Collection fieldAndFormats) { + Collection fieldAndFormats) { + Set nestedMappingPaths = context.hasNested() + ? context.nestedMappings().stream().map(ObjectMapper::name).collect(Collectors.toSet()) + : Collections.emptySet(); + return create(context, fieldAndFormats, nestedMappingPaths, ""); + } + + private static FieldFetcher create(SearchExecutionContext context, + Collection fieldAndFormats, Set nestedMappingsInScope, String nestedScopePath) { + // here we only need the nested paths that are closes to the root, e.g. only "foo" if also "foo.bar" is present. + // the remaining nested field paths are handled recursively + Set nestedParentPaths = getParentPaths(nestedMappingsInScope, context); // Using a LinkedHashMap so fields are returned in the order requested. // We won't formally guarantee this but but its good for readability of the response Map fieldContexts = new LinkedHashMap<>(); List unmappedFetchPattern = new ArrayList<>(); - boolean includeUnmapped = false; for (FieldAndFormat fieldAndFormat : fieldAndFormats) { String fieldPattern = fieldAndFormat.field; if (fieldAndFormat.includeUnmapped != null && fieldAndFormat.includeUnmapped) { unmappedFetchPattern.add(fieldAndFormat.field); - includeUnmapped = true; } - String format = fieldAndFormat.format; Collection concreteFields = context.simpleMatchToIndexNames(fieldPattern); for (String field : concreteFields) { @@ -57,49 +69,84 @@ public static FieldFetcher create(SearchExecutionContext context, if (ft == null || context.isMetadataField(field)) { continue; } - ValueFetcher valueFetcher = ft.valueFetcher(context, format); - fieldContexts.put(field, new FieldContext(field, valueFetcher)); + if (field.startsWith(nestedScopePath) == false) { + // this field is out of scope for this FieldFetcher (its likely nested) so ignore + continue; + } + String nestedParentPath = null; + if (nestedParentPaths.isEmpty() == false) { + // try to find the shortest nested parent path for this field + for (String nestedFieldPath : nestedParentPaths) { + if (field.startsWith(nestedFieldPath)) { + nestedParentPath = nestedFieldPath; + break; + } + } + } + // only add concrete fields if they are not beneath a known nested field + if (nestedParentPath == null) { + ValueFetcher valueFetcher = ft.valueFetcher(context, fieldAndFormat.format); + fieldContexts.put(field, new FieldContext(field, valueFetcher)); + } } } - CharacterRunAutomaton unmappedFetchAutomaton = new CharacterRunAutomaton(Automata.makeEmpty()); + + // create a new nested value fetcher for patterns under nested field + for (String nestedFieldPath : nestedParentPaths) { + // We construct a field fetcher that narrows the allowed lookup scope to everything beneath its nested field path. + // We also need to remove this nested field path and everything beneath it from the list of available nested fields before + // creating this internal field fetcher to avoid infinite loops on this recursion + Set narrowedScopeNestedMappings = nestedMappingsInScope.stream() + .filter(s -> nestedParentPaths.contains(s) == false) + .collect(Collectors.toSet()); + + FieldFetcher nestedSubFieldFetcher = FieldFetcher.create( + context, + fieldAndFormats, + narrowedScopeNestedMappings, + nestedFieldPath + ); + + // add a special ValueFetcher that filters source and collects its subfields + fieldContexts.put( + nestedFieldPath, + new FieldContext(nestedFieldPath, new NestedValueFetcher(nestedFieldPath, nestedSubFieldFetcher)) + ); + } + + CharacterRunAutomaton unmappedFieldsFetchAutomaton = null; if (unmappedFetchPattern.isEmpty() == false) { - unmappedFetchAutomaton = new CharacterRunAutomaton( + unmappedFieldsFetchAutomaton = new CharacterRunAutomaton( Regex.simpleMatchToAutomaton(unmappedFetchPattern.toArray(new String[unmappedFetchPattern.size()])) ); } - return new FieldFetcher(fieldContexts, unmappedFetchAutomaton, includeUnmapped); + return new FieldFetcher(fieldContexts, unmappedFieldsFetchAutomaton); } private final Map fieldContexts; - private final CharacterRunAutomaton unmappedFetchAutomaton; - private final boolean includeUnmapped; + private final CharacterRunAutomaton unmappedFieldsFetchAutomaton; private FieldFetcher( Map fieldContexts, - CharacterRunAutomaton unmappedFetchAutomaton, - boolean includeUnmapped + @Nullable CharacterRunAutomaton unmappedFieldsFetchAutomaton ) { this.fieldContexts = fieldContexts; - this.unmappedFetchAutomaton = unmappedFetchAutomaton; - this.includeUnmapped = includeUnmapped; + this.unmappedFieldsFetchAutomaton = unmappedFieldsFetchAutomaton; } public Map fetch(SourceLookup sourceLookup, Set ignoredFields) throws IOException { + assert ignoredFields != null; Map documentFields = new HashMap<>(); for (FieldContext context : fieldContexts.values()) { String field = context.fieldName; - if (ignoredFields.contains(field)) { - continue; - } ValueFetcher valueFetcher = context.valueFetcher; - List parsedValues = valueFetcher.fetchValues(sourceLookup); - + List parsedValues = valueFetcher.fetchValues(sourceLookup, ignoredFields); if (parsedValues.isEmpty() == false) { documentFields.put(field, new DocumentField(field, parsedValues)); } } - if (this.includeUnmapped) { + if (this.unmappedFieldsFetchAutomaton != null) { collectUnmapped(documentFields, sourceLookup.source(), "", 0); } return documentFields; @@ -109,7 +156,10 @@ private void collectUnmapped(Map documentFields, Map documentFields, Map) value, currentPath + ".", - step(this.unmappedFetchAutomaton, ".", currentState) + step(this.unmappedFieldsFetchAutomaton, ".", currentState) ); } else if (value instanceof List) { // iterate through list values collectUnmappedList(documentFields, (List) value, currentPath, currentState); } else { // we have a leaf value - if (this.unmappedFetchAutomaton.isAccept(currentState) && this.fieldContexts.containsKey(currentPath) == false) { + if (this.unmappedFieldsFetchAutomaton.isAccept(currentState)) { if (value != null) { DocumentField currentEntry = documentFields.get(currentPath); if (currentEntry == null) { @@ -151,12 +201,12 @@ private void collectUnmappedList(Map documentFields, Iter documentFields, (Map) value, parentPath + ".", - step(this.unmappedFetchAutomaton, ".", lastState) + step(this.unmappedFieldsFetchAutomaton, ".", lastState) ); } else if (value instanceof List) { // weird case, but can happen for objects with "enabled" : "false" collectUnmappedList(documentFields, (List) value, parentPath, lastState); - } else if (this.unmappedFetchAutomaton.isAccept(lastState) && this.fieldContexts.containsKey(parentPath) == false) { + } else if (this.unmappedFieldsFetchAutomaton.isAccept(lastState) && this.fieldContexts.containsKey(parentPath) == false) { list.add(value); } } @@ -170,6 +220,19 @@ private void collectUnmappedList(Map documentFields, Iter } } + private static Set getParentPaths(Set nestedPathsInScope, SearchExecutionContext context) { + Set parentPaths = new HashSet<>(); + for (String candidate : nestedPathsInScope) { + String nestedParent = context.getNestedParent(candidate); + // if the candidate has no nested parent itself, its a minimal parent path + // if the candidate has a parent which is out of scope this means it minimal itself + if (nestedParent == null || nestedPathsInScope.contains(nestedParent) == false) { + parentPaths.add(candidate); + } + } + return parentPaths; + } + private static int step(CharacterRunAutomaton automaton, String key, int state) { for (int i = 0; state != -1 && i < key.length(); ++i) { state = automaton.step(state, key.charAt(i)); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java index 8022c223ac80d..aaa18461633d2 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java @@ -50,7 +50,7 @@ public static List loadFieldValues(MappedFieldType fieldType, } ValueFetcher fetcher = fieldType.valueFetcher(searchContext, null); fetcher.setNextReader(hitContext.readerContext()); - return fetcher.fetchValues(hitContext.sourceLookup()); + return fetcher.fetchValues(hitContext.sourceLookup(), Collections.emptySet()); } public static class Encoders { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 2ce004f7d8c4f..8f9904ffca9dd 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -32,6 +32,7 @@ import java.util.Set; import static java.util.Collections.emptyMap; +import static org.elasticsearch.common.xcontent.ObjectPath.eval; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; @@ -495,6 +496,97 @@ public void testSimpleUnmappedArrayWithObjects() throws IOException { assertThat(field.getValues(), hasItems(1, 2, "foo")); } + public void testNestedFields() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("_doc") + .startObject("properties") + .startObject("f1") + .field("type", "keyword") + .endObject() + .startObject("obj") + .field("type", "nested") + .startObject("properties") + .startObject("f2").field("type", "keyword").endObject() + .startObject("f3").field("type", "keyword").endObject() + .startObject("inner_nested") + .field("type", "nested") + .startObject("properties") + .startObject("f4").field("type", "keyword").endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + MapperService mapperService = createMapperService(mapping); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .field("f1", "value1") + .startArray("obj") + .startObject() + .field("f2", "value2a") + .startObject("inner_nested") + .field("f4", "value4a") + .endObject() + .endObject() + .startObject() + .field("f2", "value2b") + .field("f3", "value3b") + .startObject("inner_nested") + .field("f4", "value4b") + .endObject() + .endObject() + .endArray() + .endObject(); + + Map fields = fetchFields(mapperService, source, fieldAndFormatList("*", null, false), null); + assertEquals(2, fields.size()); + assertThat(fields.keySet(), containsInAnyOrder("f1", "obj")); + assertEquals("value1", fields.get("f1").getValue()); + List obj = fields.get("obj").getValues(); + assertEquals(2, obj.size()); + Object obj0 = obj.get(0); + assertEquals(2, ((Map) obj0).size()); + assertEquals("value2a", eval("f2.0", obj0)); + assertNull(eval("f3", obj0)); + assertEquals("value4a", eval("inner_nested.0.f4.0", obj0)); + + Object obj1 = obj.get(1); + assertEquals(3, ((Map) obj1).size()); + assertEquals("value2b", eval("f2.0", obj1)); + assertEquals("value3b", eval("f3.0", obj1)); + assertEquals("value4b", eval("inner_nested.0.f4.0", obj1)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("obj*", null, false), null); + assertEquals(1, fields.size()); + assertThat(fields.keySet(), containsInAnyOrder("obj")); + obj = fields.get("obj").getValues(); + assertEquals(2, ((Map) obj.get(0)).size()); + obj0 = obj.get(0); + assertEquals(2, ((Map) obj0).size()); + assertEquals("value2a", eval("f2.0", obj0)); + assertNull(eval("f3", obj0)); + assertEquals("value4a", eval("inner_nested.0.f4.0", obj0)); + + obj1 = obj.get(1); + assertEquals(3, ((Map) obj1).size()); + assertEquals("value2b", eval("f2.0", obj1)); + assertEquals("value3b", eval("f3.0", obj1)); + assertEquals("value4b", eval("inner_nested.0.f4.0", obj1)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("obj*", null, false), null); + assertEquals(1, fields.size()); + assertThat(fields.keySet(), containsInAnyOrder("obj")); + obj = fields.get("obj").getValues(); + assertEquals(2, obj.size()); + obj0 = obj.get(0); + assertEquals("value4a", eval("inner_nested.0.f4.0", obj0)); + obj1 = obj.get(1); + assertEquals("value4b", eval("inner_nested.0.f4.0", obj1)); + } + public void testUnmappedFieldsInsideObject() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("_doc") @@ -694,7 +786,7 @@ private static Map fetchFields( SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSource(BytesReference.bytes(source)); - FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(mapperService), null, fields); + FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(mapperService), fields); return fieldFetcher.fetch(sourceLookup, ignoreFields != null ? ignoreFields : Collections.emptySet()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index 59089c6773b9b..bd81a335a648d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -46,6 +46,6 @@ public static List fetchSourceValue(MappedFieldType fieldType, Object sourceV ValueFetcher fetcher = fieldType.valueFetcher(searchExecutionContext, format); SourceLookup lookup = new SourceLookup(); lookup.setSource(Collections.singletonMap(field, sourceValue)); - return fetcher.fetchValues(lookup); + return fetcher.fetchValues(lookup, Collections.emptySet()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 15bb1423b5754..1e8eec85ea04d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -298,7 +298,7 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi LeafReaderContext context = searcher.getIndexReader().leaves().get(0); lookup.source().setSegmentAndDocument(context, 0); valueFetcher.setNextReader(context); - result.set(valueFetcher.fetchValues(lookup.source())); + result.set(valueFetcher.fetchValues(lookup.source(), Collections.emptySet())); }); return result.get(); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 3d33a4143f34a..7222ae99aac59 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -138,8 +138,8 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } return value == null - ? lookup -> Collections.emptyList() - : lookup -> Collections.singletonList(value); + ? (lookup, ignoredFields) -> Collections.emptyList() + : (lookup, ignoredFields) -> Collections.singletonList(value); } @Override diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index 05d901ac2c4ca..33eb699b0f296 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -113,13 +113,13 @@ public void testFetchValue() throws Exception { SourceLookup nullValueLookup = new SourceLookup(); nullValueLookup.setSource(Collections.singletonMap("field", null)); - assertTrue(fetcher.fetchValues(missingValueLookup).isEmpty()); - assertTrue(fetcher.fetchValues(nullValueLookup).isEmpty()); + assertTrue(fetcher.fetchValues(missingValueLookup, Collections.emptySet()).isEmpty()); + assertTrue(fetcher.fetchValues(nullValueLookup, Collections.emptySet()).isEmpty()); MappedFieldType valued = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", "foo"); fetcher = valued.valueFetcher(null, null); - assertEquals(Collections.singletonList("foo"), fetcher.fetchValues(missingValueLookup)); - assertEquals(Collections.singletonList("foo"), fetcher.fetchValues(nullValueLookup)); + assertEquals(Collections.singletonList("foo"), fetcher.fetchValues(missingValueLookup, Collections.emptySet())); + assertEquals(Collections.singletonList("foo"), fetcher.fetchValues(nullValueLookup, Collections.emptySet())); } } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java index 983a66f783f78..8ff06d416de25 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java @@ -265,7 +265,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { // This is an internal field but it can match a field pattern so we return an empty list. - return lookup -> Collections.emptyList(); + return (lookup, ignoredFields) -> Collections.emptyList(); } } diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 18b27d8064ba9..ce5ef9ff0fc59 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -62,6 +62,9 @@ subprojects { 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', //completion suggester does not return options when the context field is a geo_point runtime field 'suggest/30_context/Multi contexts should work', + + //there is something wrong when using dotted document syntax here, passes in main yaml tests + 'search/330_fetch_fields/Test nested field inside object structure', /////// TO FIX /////// /////// NOT SUPPORTED /////// diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml index 13de808e927bf..bd6f9eac658bd 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml @@ -188,6 +188,5 @@ - match: { hits.total.value: 1 } - length: { hits.hits: 1 } - - length: { hits.hits.0.fields: 2 } + - length: { hits.hits.0.fields: 1 } - match: { hits.hits.0.fields.flattened: [ { "some_field": "some_value" } ] } - - match: { hits.hits.0.fields.flattened\.some_field: [ "some_value" ] }