Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for source_path to the field caps API. #52345

Closed
wants to merge 1 commit into from
Closed

Add support for source_path to the field caps API. #52345

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1229,11 +1229,11 @@ public void testFieldCaps() throws IOException {
assertEquals(2, ratingResponse.size());

FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities(
"rating", "keyword", true, true, new String[]{"index2"}, null, null, Collections.emptyMap());
"rating", "keyword", true, true, new String[]{"index2"}, null, null, Collections.emptyMap(), Collections.emptyList());
assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword"));

FieldCapabilities expectedLongCapabilities = new FieldCapabilities(
"rating", "long", true, true, new String[]{"index1"}, null, null, Collections.emptyMap());
"rating", "long", true, true, new String[]{"index1"}, null, null, Collections.emptyMap(), Collections.emptyList());
assertEquals(expectedLongCapabilities, ratingResponse.get("long"));

// Check the capabilities for the 'field' field.
Expand All @@ -1242,7 +1242,7 @@ public void testFieldCaps() throws IOException {
assertEquals(1, fieldResponse.size());

FieldCapabilities expectedTextCapabilities = new FieldCapabilities(
"field", "text", true, false, null, null, null, Collections.emptyMap());
"field", "text", true, false, null, null, null, Collections.emptyMap(), Collections.emptyList());
assertEquals(expectedTextCapabilities, fieldResponse.get("text"));
}

Expand Down
76 changes: 76 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_source_path.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
---
setup:
- skip:
version: " - 7.99.99"
reason: "source_path is currently only supported on 8.0."

- do:
indices.create:
index: index1
body:
mappings:
properties:
field:
type: text
new_field:
type: alias
path: field
copy_to_field1:
type: text
copy_to: field

- do:
indices.create:
index: index2
body:
mappings:
properties:
field:
type: text
new_field:
type: alias
path: field
copy_to_field2:
type: text
copy_to: field

- do:
indices.create:
index: index3
body:
mappings:
properties:
field:
type: text
new_field:
type: text
copy_to_field2:
type: text
copy_to: field

---
"Merge source path information across multiple indices":
- do:
field_caps:
index: index1
fields: ["field", "new_field"]

- match: {"fields.field.text.source_path.0.indices": ["index1"]}
- match: {"fields.field.text.source_path.0.paths": ["copy_to_field1"]}

- match: {"fields.new_field.text.source_path.0.indices": ["index1"]}
- match: {"fields.new_field.text.source_path.0.paths": ["field"]}

- do:
field_caps:
index: index1,index2,index3
fields: ["field", "new_field"]

- match: {"fields.field.text.source_path.0.indices": ["index1"]}
- match: {"fields.field.text.source_path.0.paths": ["copy_to_field1"]}

- match: {"fields.field.text.source_path.1.indices": ["index2", "index3"]}
- match: {"fields.field.text.source_path.1.paths": ["copy_to_field2"]}

- match: {"fields.new_field.text.source_path.0.indices": ["index1", "index2"]}
- match: {"fields.new_field.text.source_path.0.paths": ["field"]}
138 changes: 132 additions & 6 deletions server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.function.Function;
import java.util.stream.Collectors;

Expand All @@ -56,6 +58,8 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
private static final ParseField NON_SEARCHABLE_INDICES_FIELD = new ParseField("non_searchable_indices");
private static final ParseField NON_AGGREGATABLE_INDICES_FIELD = new ParseField("non_aggregatable_indices");
private static final ParseField META_FIELD = new ParseField("meta");
private static final ParseField SOURCE_PATH_FIELD = new ParseField("source_path");


private final String name;
private final String type;
Expand All @@ -67,6 +71,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
private final String[] nonAggregatableIndices;

private final Map<String, Set<String>> meta;
private final List<SourcePath> sourcePath;

/**
* Constructor for a set of indices.
Expand All @@ -81,13 +86,15 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
* @param nonAggregatableIndices The list of indices where this field is not aggregatable,
* or null if the field is aggregatable in all indices.
* @param meta Merged metadata across indices.
* @param sourcePath Merged source paths across indices.
*/
public FieldCapabilities(String name, String type,
boolean isSearchable, boolean isAggregatable,
String[] indices,
String[] nonSearchableIndices,
String[] nonAggregatableIndices,
Map<String, Set<String>> meta) {
Map<String, Set<String>> meta,
List<SourcePath> sourcePath) {
this.name = name;
this.type = type;
this.isSearchable = isSearchable;
Expand All @@ -96,6 +103,7 @@ public FieldCapabilities(String name, String type,
this.nonSearchableIndices = nonSearchableIndices;
this.nonAggregatableIndices = nonAggregatableIndices;
this.meta = Objects.requireNonNull(meta);
this.sourcePath = sourcePath;
}

FieldCapabilities(StreamInput in) throws IOException {
Expand All @@ -111,6 +119,11 @@ public FieldCapabilities(String name, String type,
} else {
meta = Collections.emptyMap();
}
if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
sourcePath = in.readList(SourcePath::new);
} else {
sourcePath = Collections.emptyList();
}
}

@Override
Expand All @@ -125,6 +138,9 @@ public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
out.writeMap(meta, StreamOutput::writeString, (o, set) -> o.writeCollection(set, StreamOutput::writeString));
}
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
out.writeCollection(sourcePath, (o, path) -> path.writeTo(o));
}
}

@Override
Expand Down Expand Up @@ -153,6 +169,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endObject();
}
if (sourcePath.isEmpty() == false) {
builder.field(SOURCE_PATH_FIELD.getPreferredName(), sourcePath);
}
builder.endObject();
return builder;
}
Expand All @@ -172,7 +191,8 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser)
a[3] != null ? ((List<String>) a[3]).toArray(new String[0]) : null,
a[4] != null ? ((List<String>) a[4]).toArray(new String[0]) : null,
a[5] != null ? ((List<String>) a[5]).toArray(new String[0]) : null,
a[6] != null ? ((Map<String, Set<String>>) a[6]) : Collections.emptyMap()));
a[6] != null ? ((Map<String, Set<String>>) a[6]) : Collections.emptyMap(),
a[7] != null ? ((List<SourcePath>) a[7]): Collections.emptyList()));

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD);
Expand All @@ -183,6 +203,8 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser)
PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD);
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(),
(parser, context) -> parser.map(HashMap::new, p -> Set.copyOf(p.list())), META_FIELD);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(),
(parser, context) -> SourcePath.fromXContent(parser), SOURCE_PATH_FIELD);
}

/**
Expand Down Expand Up @@ -244,6 +266,13 @@ public Map<String, Set<String>> meta() {
return meta;
}

/**
* A merged list of source paths across indices.
*/
public List<SourcePath> sourcePath() {
return sourcePath;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand All @@ -256,12 +285,13 @@ public boolean equals(Object o) {
Arrays.equals(indices, that.indices) &&
Arrays.equals(nonSearchableIndices, that.nonSearchableIndices) &&
Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices) &&
Objects.equals(meta, that.meta);
Objects.equals(meta, that.meta) &&
Objects.equals(sourcePath, that.sourcePath);
}

@Override
public int hashCode() {
int result = Objects.hash(name, type, isSearchable, isAggregatable, meta);
int result = Objects.hash(name, type, isSearchable, isAggregatable, meta, sourcePath);
result = 31 * result + Arrays.hashCode(indices);
result = 31 * result + Arrays.hashCode(nonSearchableIndices);
result = 31 * result + Arrays.hashCode(nonAggregatableIndices);
Expand All @@ -281,27 +311,39 @@ static class Builder {
private List<IndexCaps> indiceList;
private Map<String, Set<String>> meta;

// Maps from a sorted list of paths to index names. This allows us to detect when multiple
// indices have the same source path information, and combine them into a single entry.
private Map<SortedSet<String>, Set<String>> sourcePaths;

Builder(String name, String type) {
this.name = name;
this.type = type;
this.isSearchable = true;
this.isAggregatable = true;
this.indiceList = new ArrayList<>();
this.meta = new HashMap<>();
this.sourcePaths = new HashMap<>();
}

/**
* Collect the field capabilities for an index.
*/
void add(String index, boolean search, boolean agg, Map<String, String> meta) {
void add(String index, boolean search, boolean agg, Map<String, String> meta, Set<String> sourcePath) {
IndexCaps indexCaps = new IndexCaps(index, search, agg);
indiceList.add(indexCaps);
this.isSearchable &= search;
this.isAggregatable &= agg;

for (Map.Entry<String, String> entry : meta.entrySet()) {
this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>())
.add(entry.getValue());
}

if (sourcePath.isEmpty() == false) {
SortedSet<String> sortedPaths = Collections.unmodifiableSortedSet(new TreeSet<>(sourcePath));
this.sourcePaths.computeIfAbsent(sortedPaths, key -> new HashSet<>())
.add(index);
}
}

List<String> getIndices() {
Expand Down Expand Up @@ -347,11 +389,95 @@ FieldCapabilities build(boolean withIndices) {
nonAggregatableIndices = null;
}
final Function<Map.Entry<String, Set<String>>, Set<String>> entryValueFunction = Map.Entry::getValue;

Map<String, Set<String>> immutableMeta = meta.entrySet().stream()
.collect(Collectors.toUnmodifiableMap(
Map.Entry::getKey, entryValueFunction.andThen(Set::copyOf)));

List<SourcePath> immutableSourcePath = sourcePaths.entrySet().stream()
.map(entry -> new SourcePath(
entry.getValue().toArray(new String[0]),
entry.getKey().toArray(new String[0])))
.sorted(Comparator.comparing(sourcePath -> sourcePath.indices()[0]))
.collect(Collectors.toList());

return new FieldCapabilities(name, type, isSearchable, isAggregatable,
indices, nonSearchableIndices, nonAggregatableIndices, immutableMeta);
indices, nonSearchableIndices, nonAggregatableIndices,
immutableMeta, immutableSourcePath);
}
}

public static class SourcePath implements Writeable, ToXContentObject {
private static final ParseField INDICES_FIELD = new ParseField("indices");
private static final ParseField PATHS_FIELD = new ParseField("paths");

@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<SourcePath, Void> PARSER = new ConstructingObjectParser<>(
"source_path", true,
(a, name) -> new SourcePath(
((List<String>) a[0]).toArray(new String[0]),
((List<String>) a[1]).toArray(new String[0])));

static {
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD);
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PATHS_FIELD);
}

private final String[] indices;
private final String[] paths;

public SourcePath(String[] indices, String[] paths) {
this.indices = Arrays.copyOf(indices, indices.length);
this.paths = Arrays.copyOf(paths, paths.length);
Arrays.sort(this.indices);
Arrays.sort(this.paths);
}

SourcePath(StreamInput in) throws IOException {
this.indices = in.readStringArray();
this.paths = in.readStringArray();
}

public String[] indices() {
return indices;
}

public String[] paths() {
return paths;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeStringArray(indices);
out.writeStringArray(paths);
}

public static SourcePath fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject()
.field(INDICES_FIELD.getPreferredName(), indices)
.field(PATHS_FIELD.getPreferredName(), paths)
.endObject();
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SourcePath that = (SourcePath) o;
return Arrays.equals(indices, that.indices) &&
Arrays.equals(paths, that.paths);
}

@Override
public int hashCode() {
int result = Arrays.hashCode(indices);
result = 31 * result + Arrays.hashCode(paths);
return result;
}
}

Expand Down
Loading