diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index b08b045d287c0..d51a92ea00fc5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -20,6 +20,7 @@ package org.elasticsearch.client; import org.apache.http.Header; +import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; @@ -38,6 +39,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; @@ -134,11 +137,34 @@ public PutMappingResponse putMapping(PutMappingRequest putMappingRequest, Header * Put Mapping API on elastic.co */ public void putMappingAsync(PutMappingRequest putMappingRequest, ActionListener listener, - Header... headers) { + Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(putMappingRequest, RequestConverters::putMapping, PutMappingResponse::fromXContent, listener, emptySet(), headers); } + /** + * Retrieves the mappings on an index or indices using the Get Mapping API + *

+ * See + * Get Mapping API on elastic.co + */ + public GetMappingsResponse getMappings(GetMappingsRequest getMappingsRequest, Header... headers) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest, RequestConverters::getMappings, + GetMappingsResponse::fromXContent, emptySet(), headers); + } + + /** + * Asynchronously retrieves the mappings on an index on indices using the Get Mapping API + *

+ * See + * Get Mapping API on elastic.co + */ + public void getMappingsAsync(GetMappingsRequest getMappingsRequest, ActionListener listener, + Header... headers) { + restHighLevelClient.performRequestAsyncAndParseEntity(getMappingsRequest, RequestConverters::getMappings, + GetMappingsResponse::fromXContent, listener, emptySet(), headers); + } + /** * Updates aliases using the Index Aliases API *

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 0a8323a29f5c2..cceb208e9268e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -45,6 +45,7 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -198,6 +199,19 @@ static Request putMapping(PutMappingRequest putMappingRequest) throws IOExceptio return request; } + static Request getMappings(GetMappingsRequest getMappingsRequest) throws IOException { + String[] indices = getMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.indices(); + String[] types = getMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.types(); + + Request request = new Request(HttpGet.METHOD_NAME, endpoint(indices, "_mapping", types)); + + Params parameters = new Params(request); + parameters.withMasterTimeout(getMappingsRequest.masterNodeTimeout()); + parameters.withIndicesOptions(getMappingsRequest.indicesOptions()); + parameters.withLocal(getMappingsRequest.local()); + return request; + } + static Request refresh(RefreshRequest refreshRequest) { String[] indices = refreshRequest.indices() == null ? Strings.EMPTY_ARRAY : refreshRequest.indices(); Request request = new Request(HttpPost.METHOD_NAME, endpoint(indices, "_refresh")); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 120ba65673b36..b7fccd41c36bc 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -42,6 +42,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; @@ -80,6 +82,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; @@ -329,6 +332,42 @@ public void testPutMapping() throws IOException { } } + public void testGetMapping() throws IOException { + String indexName = "test"; + createIndex(indexName, Settings.EMPTY); + + PutMappingRequest putMappingRequest = new PutMappingRequest(indexName); + putMappingRequest.type("_doc"); + XContentBuilder mappingBuilder = JsonXContent.contentBuilder(); + mappingBuilder.startObject().startObject("properties").startObject("field"); + mappingBuilder.field("type", "text"); + mappingBuilder.endObject().endObject().endObject(); + putMappingRequest.source(mappingBuilder); + + PutMappingResponse putMappingResponse = + execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync); + assertTrue(putMappingResponse.isAcknowledged()); + + Map getIndexResponse = getAsMap(indexName); + assertEquals("text", XContentMapValues.extractValue(indexName + ".mappings._doc.properties.field.type", getIndexResponse)); + + GetMappingsRequest request = new GetMappingsRequest() + .indices(indexName) + .types("_doc"); + + GetMappingsResponse getMappingsResponse = + execute(request, highLevelClient().indices()::getMappings, highLevelClient().indices()::getMappingsAsync); + + Map mappings = getMappingsResponse.getMappings().get(indexName).get("_doc").sourceAsMap(); + Map type = new HashMap<>(); + type.put("type", "text"); + Map field = new HashMap<>(); + field.put("field", type); + Map expected = new HashMap<>(); + expected.put("properties", field); + assertThat(mappings, equalTo(expected)); + } + public void testDeleteIndex() throws IOException { { // Delete index if exists diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 6a7da03c3b654..2645846341f4b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -47,6 +47,7 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -413,6 +414,47 @@ public void testPutMapping() throws IOException { assertToXContentBody(putMappingRequest, request.getEntity()); } + public void testGetMapping() throws IOException { + GetMappingsRequest getMappingRequest = new GetMappingsRequest(); + + String[] indices = Strings.EMPTY_ARRAY; + if (randomBoolean()) { + indices = randomIndicesNames(0, 5); + getMappingRequest.indices(indices); + } else if (randomBoolean()) { + getMappingRequest.indices((String[]) null); + } + + String type = null; + if (randomBoolean()) { + type = randomAlphaOfLengthBetween(3, 10); + getMappingRequest.types(type); + } else if (randomBoolean()) { + getMappingRequest.types((String[]) null); + } + + Map expectedParams = new HashMap<>(); + + setRandomIndicesOptions(getMappingRequest::indicesOptions, getMappingRequest::indicesOptions, expectedParams); + setRandomMasterTimeout(getMappingRequest, expectedParams); + setRandomLocal(getMappingRequest, expectedParams); + + Request request = RequestConverters.getMappings(getMappingRequest); + StringJoiner endpoint = new StringJoiner("/", "/", ""); + String index = String.join(",", indices); + if (Strings.hasLength(index)) { + endpoint.add(index); + } + endpoint.add("_mapping"); + if (type != null) { + endpoint.add(type); + } + assertThat(endpoint.toString(), equalTo(request.getEndpoint())); + + assertThat(expectedParams, equalTo(request.getParameters())); + assertThat(HttpGet.METHOD_NAME, equalTo(request.getMethod())); + } + public void testDeleteIndex() { String[] indices = randomIndicesNames(0, 5); DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indices); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 0ec3eeb986429..afae4a5a7a0fe 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -41,6 +41,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; @@ -64,6 +66,8 @@ import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.SyncedFlushResponse; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -81,6 +85,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static org.hamcrest.Matchers.equalTo; + /** * This class is used to generate the Java Indices API documentation. * You need to wrap your code between two tags like: @@ -532,17 +538,17 @@ public void testPutMappingAsync() throws Exception { // tag::put-mapping-execute-listener ActionListener listener = - new ActionListener() { - @Override - public void onResponse(PutMappingResponse putMappingResponse) { - // <1> - } + new ActionListener() { + @Override + public void onResponse(PutMappingResponse putMappingResponse) { + // <1> + } - @Override - public void onFailure(Exception e) { - // <2> - } - }; + @Override + public void onFailure(Exception e) { + // <2> + } + }; // end::put-mapping-execute-listener // Replace the empty listener by a blocking listener in test @@ -557,6 +563,133 @@ public void onFailure(Exception e) { } } + public void testGetMapping() throws IOException { + RestHighLevelClient client = highLevelClient(); + + { + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter")); + assertTrue(createIndexResponse.isAcknowledged()); + PutMappingRequest request = new PutMappingRequest("twitter"); + request.type("tweet"); + request.source( + "{\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + "}", // <1> + XContentType.JSON); + PutMappingResponse putMappingResponse = client.indices().putMapping(request); + assertTrue(putMappingResponse.isAcknowledged()); + } + + { + // tag::get-mapping-request + GetMappingsRequest request = new GetMappingsRequest(); // <1> + request.indices("twitter"); // <2> + request.types("tweet"); // <3> + // end::get-mapping-request + + // tag::get-mapping-request-masterTimeout + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> + request.masterNodeTimeout("1m"); // <2> + // end::get-mapping-request-masterTimeout + + // tag::get-mapping-request-indicesOptions + request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1> + // end::get-mapping-request-indicesOptions + + // tag::get-mapping-execute + GetMappingsResponse getMappingResponse = client.indices().getMappings(request); + // end::get-mapping-execute + + // tag::get-mapping-response + ImmutableOpenMap> allMappings = getMappingResponse.mappings(); // <1> + MappingMetaData typeMapping = allMappings.get("twitter").get("tweet"); // <2> + Map tweetMapping = typeMapping.sourceAsMap(); // <3> + // end::get-mapping-response + + Map type = new HashMap<>(); + type.put("type", "text"); + Map field = new HashMap<>(); + field.put("message", type); + Map expected = new HashMap<>(); + expected.put("properties", field); + assertThat(tweetMapping, equalTo(expected)); + } + } + + public void testGetMappingAsync() throws Exception { + final RestHighLevelClient client = highLevelClient(); + + { + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter")); + assertTrue(createIndexResponse.isAcknowledged()); + PutMappingRequest request = new PutMappingRequest("twitter"); + request.type("tweet"); + request.source( + "{\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + "}", // <1> + XContentType.JSON); + PutMappingResponse putMappingResponse = client.indices().putMapping(request); + assertTrue(putMappingResponse.isAcknowledged()); + } + + { + GetMappingsRequest request = new GetMappingsRequest(); + request.indices("twitter"); + request.types("tweet"); + + // tag::get-mapping-execute-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(GetMappingsResponse putMappingResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::get-mapping-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + final ActionListener latchListener = new LatchedActionListener<>(listener, latch); + listener = ActionListener.wrap(r -> { + ImmutableOpenMap> allMappings = r.mappings(); + MappingMetaData typeMapping = allMappings.get("twitter").get("tweet"); + Map tweetMapping = typeMapping.sourceAsMap(); + + Map type = new HashMap<>(); + type.put("type", "text"); + Map field = new HashMap<>(); + field.put("message", type); + Map expected = new HashMap<>(); + expected.put("properties", field); + assertThat(tweetMapping, equalTo(expected)); + latchListener.onResponse(r); + }, e -> { + latchListener.onFailure(e); + fail("should not fail"); + }); + + // tag::get-mapping-execute-async + client.indices().getMappingsAsync(request, listener); // <1> + // end::get-mapping-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + public void testOpenIndex() throws Exception { RestHighLevelClient client = highLevelClient(); diff --git a/docs/java-rest/high-level/indices/get_mappings.asciidoc b/docs/java-rest/high-level/indices/get_mappings.asciidoc new file mode 100644 index 0000000000000..f3506b6bcda57 --- /dev/null +++ b/docs/java-rest/high-level/indices/get_mappings.asciidoc @@ -0,0 +1,80 @@ +[[java-rest-high-get-mappings]] +=== Get Mappings API + +[[java-rest-high-get-mappings-request]] +==== Get Mappings Request + +A `GetMappingsRequest` can have an optional list of indices and optional list of types: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-mapping-request] +-------------------------------------------------- +<1> An empty request that will return all indices and types +<2> Setting the indices to fetch mapping for +<3> The types to be returned + +==== Optional arguments +The following arguments can also optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-mapping-request-masterTimeout] +-------------------------------------------------- +<1> Timeout to connect to the master node as a `TimeValue` +<2> Timeout to connect to the master node as a `String` + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-mapping-request-indicesOptions] +-------------------------------------------------- +<1> Options for expanding indices names + +[[java-rest-high-get-mappings-sync]] +==== Synchronous Execution + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-mapping-execute] +-------------------------------------------------- + +[[java-rest-high-get-mapping-async]] +==== Asynchronous Execution + +The asynchronous execution of a get mappings request requires both the +`GetMappingsRequest` instance and an `ActionListener` instance to be passed to +the asynchronous method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-mapping-execute-async] +-------------------------------------------------- +<1> The `GetMappingsRequest` to execute and the `ActionListener` to use when the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method if +the execution successfully completed or using the `onFailure` method if it +failed. + +A typical listener for `GetMappingsResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-mapping-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument + +[[java-rest-high-get-mapping-response]] +==== Get Mappings Response + +The returned `GetMappingsResponse` allows to retrieve information about the +executed operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-mapping-response] +-------------------------------------------------- +<1> Returning all indices' mappings +<2> Retrieving the mappings for a particular index and type +<3> Getting the mappings for the "tweet" as a Java Map diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 35d8b8901b4e7..8694a624654a5 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -95,6 +95,7 @@ include::indices/clear_cache.asciidoc[] include::indices/force_merge.asciidoc[] include::indices/rollover.asciidoc[] include::indices/put_mapping.asciidoc[] +include::indices/get_mappings.asciidoc[] include::indices/update_aliases.asciidoc[] include::indices/exists_alias.asciidoc[] include::indices/put_settings.asciidoc[] diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java index fe41352741e71..b6164c2696735 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -140,6 +141,42 @@ public void testReadMapStrings() throws IOException { assertThat(map.size(), equalTo(0)); } + public void testMap() throws IOException { + String source = "{\"i\": {\"_doc\": {\"f1\": {\"type\": \"text\", \"analyzer\": \"english\"}, " + + "\"f2\": {\"type\": \"object\", \"properties\": {\"sub1\": {\"type\": \"keyword\", \"foo\": 17}}}}}}"; + Map f1 = new HashMap<>(); + f1.put("type", "text"); + f1.put("analyzer", "english"); + + Map sub1 = new HashMap<>(); + sub1.put("type", "keyword"); + sub1.put("foo", 17); + + Map properties = new HashMap<>(); + properties.put("sub1", sub1); + + Map f2 = new HashMap<>(); + f2.put("type", "object"); + f2.put("properties", properties); + + Map doc = new HashMap<>(); + doc.put("f1", f1); + doc.put("f2", f2); + + Map expected = new HashMap<>(); + expected.put("_doc", doc); + + Map i = new HashMap<>(); + i.put("i", expected); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) { + XContentParser.Token token = parser.nextToken(); + assertThat(token, equalTo(XContentParser.Token.START_OBJECT)); + Map map = parser.map(); + assertThat(map, equalTo(i)); + } + } + private Map readMapStrings(String source) throws IOException { try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) { XContentParser.Token token = parser.nextToken(); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json index c3c0622844bb1..e29ed43b3e451 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json @@ -30,6 +30,10 @@ "default" : "open", "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both." }, + "master_timeout": { + "type" : "time", + "description" : "Specify timeout for connection to master" + }, "local": { "type": "boolean", "description": "Return local information, do not retrieve the state from master node (default: false)" diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index 12975c765d094..d21261abad89e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -20,15 +20,31 @@ package org.elasticsearch.action.admin.indices.mapping.get; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.Map; -public class GetMappingsResponse extends ActionResponse { +public class GetMappingsResponse extends ActionResponse implements ToXContentFragment { + + private static final ParseField MAPPINGS = new ParseField("mappings"); + + private static final ObjectParser PARSER = + new ObjectParser("get-mappings", false, GetMappingsResponse::new); private ImmutableOpenMap> mappings = ImmutableOpenMap.of(); @@ -77,4 +93,94 @@ public void writeTo(StreamOutput out) throws IOException { } } } + + public static GetMappingsResponse fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + assert parser.currentToken() == XContentParser.Token.START_OBJECT; + Map parts = parser.map(); + + ImmutableOpenMap.Builder> builder = new ImmutableOpenMap.Builder<>(); + for (Map.Entry entry : parts.entrySet()) { + final String indexName = entry.getKey(); + assert entry.getValue() instanceof Map : "expected a map as type mapping, but got: " + entry.getValue().getClass(); + final Map mapping = (Map) ((Map) entry.getValue()).get(MAPPINGS.getPreferredName()); + + ImmutableOpenMap.Builder typeBuilder = new ImmutableOpenMap.Builder<>(); + for (Map.Entry typeEntry : mapping.entrySet()) { + final String typeName = typeEntry.getKey(); + assert typeEntry.getValue() instanceof Map : "expected a map as inner type mapping, but got: " + + typeEntry.getValue().getClass(); + final Map fieldMappings = (Map) typeEntry.getValue(); + MappingMetaData mmd = new MappingMetaData(typeName, fieldMappings); + typeBuilder.put(typeName, mmd); + } + builder.put(indexName, typeBuilder.build()); + } + + return new GetMappingsResponse(builder.build()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return toXContent(builder, params, true); + } + + public XContentBuilder toXContent(XContentBuilder builder, Params params, boolean includeTypeName) throws IOException { + for (final ObjectObjectCursor> indexEntry : getMappings()) { + builder.startObject(indexEntry.key); + { + if (includeTypeName == false) { + MappingMetaData mappings = null; + for (final ObjectObjectCursor typeEntry : indexEntry.value) { + if (typeEntry.key.equals("_default_") == false) { + assert mappings == null; + mappings = typeEntry.value; + } + } + if (mappings == null) { + // no mappings yet + builder.startObject(MAPPINGS.getPreferredName()).endObject(); + } else { + builder.field(MAPPINGS.getPreferredName(), mappings.sourceAsMap()); + } + } else { + builder.startObject(MAPPINGS.getPreferredName()); + { + for (final ObjectObjectCursor typeEntry : indexEntry.value) { + builder.field(typeEntry.key, typeEntry.value.sourceAsMap()); + } + } + builder.endObject(); + } + } + builder.endObject(); + } + return builder; + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public int hashCode() { + return mappings.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + GetMappingsResponse other = (GetMappingsResponse) obj; + return this.mappings.equals(other.mappings); + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 99b8215025ec7..97bd23f1a9b71 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.TypeMissingException; @@ -82,6 +83,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); getMappingsRequest.indices(indices).types(types); getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions())); + getMappingsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getMappingsRequest.masterNodeTimeout())); getMappingsRequest.local(request.paramAsBoolean("local", getMappingsRequest.local())); return channel -> client.admin().indices().getMappings(getMappingsRequest, new RestBuilderListener(channel) { @Override @@ -128,38 +130,17 @@ public RestResponse buildResponse(final GetMappingsResponse response, final XCon status = RestStatus.OK; } else { status = RestStatus.NOT_FOUND; - final String message; - if (difference.size() == 1) { - message = String.format(Locale.ROOT, "type [%s] missing", toNamesString(difference.iterator().next())); - } else { - message = String.format(Locale.ROOT, "types [%s] missing", toNamesString(difference.toArray(new String[0]))); - } + final String message = String.format(Locale.ROOT, "type" + (difference.size() == 1 ? "" : "s") + + " [%s] missing", Strings.collectionToCommaDelimitedString(difference)); builder.field("error", message); builder.field("status", status.getStatus()); } - - for (final ObjectObjectCursor> indexEntry : mappingsByIndex) { - builder.startObject(indexEntry.key); - { - builder.startObject("mappings"); - { - for (final ObjectObjectCursor typeEntry : indexEntry.value) { - builder.field(typeEntry.key, typeEntry.value.sourceAsMap()); - } - } - builder.endObject(); - } - builder.endObject(); - } + response.toXContent(builder, ToXContent.EMPTY_PARAMS); } builder.endObject(); + return new BytesRestResponse(status, builder); } }); } - - private static String toNamesString(final String... names) { - return Arrays.stream(names).collect(Collectors.joining(",")); - } - } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java new file mode 100644 index 0000000000000..0fa5ca075fa8d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java @@ -0,0 +1,153 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.mapping.get; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class GetMappingsResponseTests extends AbstractStreamableXContentTestCase { + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + public void testCheckEqualsAndHashCode() { + GetMappingsResponse resp = createTestInstance(); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(resp, r -> new GetMappingsResponse(r.mappings()), GetMappingsResponseTests::mutate); + } + + @Override + protected GetMappingsResponse doParseInstance(XContentParser parser) throws IOException { + return GetMappingsResponse.fromXContent(parser); + } + + @Override + protected GetMappingsResponse createBlankInstance() { + return new GetMappingsResponse(); + } + + private static GetMappingsResponse mutate(GetMappingsResponse original) throws IOException { + ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(original.mappings()); + String indexKey = original.mappings().keys().iterator().next().value; + + ImmutableOpenMap.Builder typeBuilder = ImmutableOpenMap.builder(original.mappings().get(indexKey)); + final String typeKey; + Iterator> iter = original.mappings().get(indexKey).keys().iterator(); + if (iter.hasNext()) { + typeKey = iter.next().value; + } else { + typeKey = "new-type"; + } + + typeBuilder.put(typeKey, new MappingMetaData("type-" + randomAlphaOfLength(6), randomFieldMapping())); + + builder.put(indexKey, typeBuilder.build()); + return new GetMappingsResponse(builder.build()); + } + + @Override + protected GetMappingsResponse mutateInstance(GetMappingsResponse instance) throws IOException { + return mutate(instance); + } + + @Override + protected GetMappingsResponse createTestInstance() { + // rarely have no types + int typeCount = rarely() ? 0 : scaledRandomIntBetween(1, 3); + List typeMappings = new ArrayList<>(typeCount); + + for (int i = 0; i < typeCount; i++) { + Map mappings = new HashMap<>(); + if (rarely() == false) { // rarely have no fields + mappings.put("field-" + i, randomFieldMapping()); + if (randomBoolean()) { + mappings.put("field2-" + i, randomFieldMapping()); + } + } + + try { + MappingMetaData mmd = new MappingMetaData("type-" + randomAlphaOfLength(5), mappings); + typeMappings.add(mmd); + } catch (IOException e) { + fail("shouldn't have failed " + e); + } + } + ImmutableOpenMap.Builder typeBuilder = ImmutableOpenMap.builder(); + typeMappings.forEach(mmd -> typeBuilder.put(mmd.type(), mmd)); + ImmutableOpenMap.Builder> indexBuilder = ImmutableOpenMap.builder(); + indexBuilder.put("index-" + randomAlphaOfLength(5), typeBuilder.build()); + GetMappingsResponse resp = new GetMappingsResponse(indexBuilder.build()); + logger.debug("--> created: {}", resp); + return resp; + } + + // Not meant to be exhaustive + private static Map randomFieldMapping() { + Map mappings = new HashMap<>(); + if (randomBoolean()) { + Map regularMapping = new HashMap<>(); + regularMapping.put("type", randomBoolean() ? "text" : "keyword"); + regularMapping.put("index", "analyzed"); + regularMapping.put("analyzer", "english"); + return regularMapping; + } else if (randomBoolean()) { + Map numberMapping = new HashMap<>(); + numberMapping.put("type", randomFrom("integer", "float", "long", "double")); + numberMapping.put("index", Objects.toString(randomBoolean())); + return numberMapping; + } else if (randomBoolean()) { + Map objMapping = new HashMap<>(); + objMapping.put("type", "object"); + objMapping.put("dynamic", "strict"); + Map properties = new HashMap<>(); + Map props1 = new HashMap<>(); + props1.put("type", randomFrom("text", "keyword")); + props1.put("analyzer", "keyword"); + properties.put("subtext", props1); + Map props2 = new HashMap<>(); + props2.put("type", "object"); + Map prop2properties = new HashMap<>(); + Map props3 = new HashMap<>(); + props3.put("type", "integer"); + props3.put("index", "false"); + prop2properties.put("subsubfield", props3); + props2.put("properties", prop2properties); + objMapping.put("properties", properties); + return objMapping; + } else { + Map plainMapping = new HashMap<>(); + plainMapping.put("type", "keyword"); + return plainMapping; + } + } +} diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 8df72aaf3e7fe..910e7e7176cc2 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -81,7 +81,6 @@ buildRestTests.expectedUnconvertedCandidates = [ 'en/rest-api/ml/validate-job.asciidoc', 'en/rest-api/security/authenticate.asciidoc', 'en/rest-api/watcher/stats.asciidoc', - 'en/security/authorization/managing-roles.asciidoc', 'en/watcher/example-watches/watching-time-series-data.asciidoc', ] diff --git a/x-pack/docs/en/security/authorization/managing-roles.asciidoc b/x-pack/docs/en/security/authorization/managing-roles.asciidoc index 83edef1a67ba4..6ee5d9d39bbf2 100644 --- a/x-pack/docs/en/security/authorization/managing-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/managing-roles.asciidoc @@ -12,6 +12,8 @@ A role is defined by the following JSON structure: "indices": [ ... ] <3> } ----- +// NOTCONSOLE + <1> A list of usernames the owners of this role can <>. <2> A list of cluster privileges. These privileges define the cluster level actions users with this role are able to execute. This field @@ -37,6 +39,8 @@ The following describes the structure of an indices permissions entry: "query": "..." <4> } ------- +// NOTCONSOLE + <1> A list of indices (or index name patterns) to which the permissions in this entry apply. <2> The index level privileges the owners of the role have on the associated @@ -77,8 +81,9 @@ The following snippet shows an example definition of a `clicks_admin` role: [source,js] ----------- +POST /_xpack/security/role/clicks_admin { - "run_as": [ "clicks_watcher_1" ] + "run_as": [ "clicks_watcher_1" ], "cluster": [ "monitor" ], "indices": [ { @@ -92,6 +97,7 @@ The following snippet shows an example definition of a `clicks_admin` role: ] } ----------- +// CONSOLE Based on the above definition, users owning the `clicks_admin` role can: diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java index a799cbe944715..3818ebcf44758 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; @@ -173,7 +174,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.endObject(); return builder; } - + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(id); @@ -336,6 +337,17 @@ public RollupJobConfig build() { if (indexPattern == null || indexPattern.isEmpty()) { throw new IllegalArgumentException("An index pattern is mandatory."); } + if (Regex.isMatchAllPattern(indexPattern)) { + throw new IllegalArgumentException("Index pattern must not match all indices (as it would match it's own rollup index"); + } + if (Regex.isSimpleMatchPattern(indexPattern)) { + if (Regex.simpleMatch(indexPattern, rollupIndex)) { + throw new IllegalArgumentException("Index pattern would match rollup index name which is not allowed."); + } + } + if (indexPattern.equals(rollupIndex)) { + throw new IllegalArgumentException("Rollup index may not be the same as the index pattern."); + } if (rollupIndex == null || rollupIndex.isEmpty()) { throw new IllegalArgumentException("A rollup index name is mandatory."); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java index 7522f474e77b2..3d82ac118f503 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java @@ -27,8 +27,9 @@ public static RollupJobConfig.Builder getRollupJob(String jobId) { builder.setId(jobId); builder.setCron(getCronString()); builder.setTimeout(new TimeValue(ESTestCase.randomIntBetween(1,100))); - builder.setIndexPattern(ESTestCase.randomAlphaOfLengthBetween(1,10)); - builder.setRollupIndex(ESTestCase.randomAlphaOfLengthBetween(1,10)); + String indexPattern = ESTestCase.randomAlphaOfLengthBetween(1,10); + builder.setIndexPattern(indexPattern); + builder.setRollupIndex("rollup_" + indexPattern); // to ensure the index pattern != rollup index builder.setGroupConfig(ConfigTestHelpers.getGroupConfig().build()); builder.setPageSize(ESTestCase.randomIntBetween(1,10)); if (ESTestCase.randomBoolean()) { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java index 24dcb323e3dc6..819a8dfa3fe9f 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java @@ -95,8 +95,8 @@ protected void masterOperation(PutRollupJobAction.Request request, ClusterState XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest() - .indices(request.getConfig().getIndexPattern()) - .fields(request.getConfig().getAllFields().toArray(new String[0])); + .indices(request.getConfig().getIndexPattern()) + .fields(request.getConfig().getAllFields().toArray(new String[0])); client.fieldCaps(fieldCapsRequest, new ActionListener() { @Override diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java index e617ad81905c5..ec7228f69ae1c 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -66,6 +67,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -276,91 +278,38 @@ static QueryBuilder rewriteQuery(QueryBuilder builder, Set jobCap rewriteQuery(((BoostingQueryBuilder)builder).positiveQuery(), jobCaps)); } else if (builder.getWriteableName().equals(DisMaxQueryBuilder.NAME)) { DisMaxQueryBuilder rewritten = new DisMaxQueryBuilder(); - ((DisMaxQueryBuilder)builder).innerQueries().forEach(query -> rewritten.add(rewriteQuery(query, jobCaps))); + ((DisMaxQueryBuilder) builder).innerQueries().forEach(query -> rewritten.add(rewriteQuery(query, jobCaps))); return rewritten; - } else if (builder.getWriteableName().equals(RangeQueryBuilder.NAME) || builder.getWriteableName().equals(TermQueryBuilder.NAME)) { - - String fieldName = builder.getWriteableName().equals(RangeQueryBuilder.NAME) - ? ((RangeQueryBuilder)builder).fieldName() - : ((TermQueryBuilder)builder).fieldName(); - - List incorrectTimeZones = new ArrayList<>(); - List rewrittenFieldName = jobCaps.stream() - // We only care about job caps that have the query's target field - .filter(caps -> caps.getFieldCaps().keySet().contains(fieldName)) - .map(caps -> { - RollupJobCaps.RollupFieldCaps fieldCaps = caps.getFieldCaps().get(fieldName); - return fieldCaps.getAggs().stream() - // For now, we only allow filtering on grouping fields - .filter(agg -> { - String type = (String)agg.get(RollupField.AGG); - - // If the cap is for a date_histo, and the query is a range, the timezones need to match - if (type.equals(DateHistogramAggregationBuilder.NAME) && builder instanceof RangeQueryBuilder) { - String timeZone = ((RangeQueryBuilder)builder).timeZone(); - - // Many range queries don't include the timezone because the default is UTC, but the query - // builder will return null so we need to set it here - if (timeZone == null) { - timeZone = DateTimeZone.UTC.toString(); - } - boolean matchingTZ = ((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName())) - .equalsIgnoreCase(timeZone); - if (matchingTZ == false) { - incorrectTimeZones.add((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName())); - } - return matchingTZ; - } - // Otherwise just make sure it's one of the three groups - return type.equals(TermsAggregationBuilder.NAME) - || type.equals(DateHistogramAggregationBuilder.NAME) - || type.equals(HistogramAggregationBuilder.NAME); - }) - // Rewrite the field name to our convention (e.g. "foo" -> "date_histogram.foo.timestamp") - .map(agg -> { - if (agg.get(RollupField.AGG).equals(DateHistogramAggregationBuilder.NAME)) { - return RollupField.formatFieldName(fieldName, (String)agg.get(RollupField.AGG), RollupField.TIMESTAMP); - } else { - return RollupField.formatFieldName(fieldName, (String)agg.get(RollupField.AGG), RollupField.VALUE); - } - }) - .collect(Collectors.toList()); - }) - .distinct() - .collect(ArrayList::new, List::addAll, List::addAll); - - if (rewrittenFieldName.isEmpty()) { - if (incorrectTimeZones.isEmpty()) { - throw new IllegalArgumentException("Field [" + fieldName + "] in [" + builder.getWriteableName() - + "] query is not available in selected rollup indices, cannot query."); - } else { - throw new IllegalArgumentException("Field [" + fieldName + "] in [" + builder.getWriteableName() - + "] query was found in rollup indices, but requested timezone is not compatible. Options include: " - + incorrectTimeZones); - } + } else if (builder.getWriteableName().equals(RangeQueryBuilder.NAME)) { + RangeQueryBuilder range = (RangeQueryBuilder) builder; + String fieldName = range.fieldName(); + // Many range queries don't include the timezone because the default is UTC, but the query + // builder will return null so we need to set it here + String timeZone = range.timeZone() == null ? DateTimeZone.UTC.toString() : range.timeZone(); + + String rewrittenFieldName = rewriteFieldName(jobCaps, RangeQueryBuilder.NAME, fieldName, timeZone); + RangeQueryBuilder rewritten = new RangeQueryBuilder(rewrittenFieldName) + .from(range.from()) + .to(range.to()) + .includeLower(range.includeLower()) + .includeUpper(range.includeUpper()); + if (range.timeZone() != null) { + rewritten.timeZone(range.timeZone()); } - - if (rewrittenFieldName.size() > 1) { - throw new IllegalArgumentException("Ambiguous field name resolution when mapping to rolled fields. Field name [" + - fieldName + "] was mapped to: [" + Strings.collectionToDelimitedString(rewrittenFieldName, ",") + "]."); + if (range.format() != null) { + rewritten.format(range.format()); } - - //Note: instanceof here to make casting checks happier - if (builder instanceof RangeQueryBuilder) { - RangeQueryBuilder rewritten = new RangeQueryBuilder(rewrittenFieldName.get(0)); - RangeQueryBuilder original = (RangeQueryBuilder)builder; - rewritten.from(original.from()); - rewritten.to(original.to()); - if (original.timeZone() != null) { - rewritten.timeZone(original.timeZone()); - } - rewritten.includeLower(original.includeLower()); - rewritten.includeUpper(original.includeUpper()); - return rewritten; - } else { - return new TermQueryBuilder(rewrittenFieldName.get(0), ((TermQueryBuilder)builder).value()); - } - + return rewritten; + } else if (builder.getWriteableName().equals(TermQueryBuilder.NAME)) { + TermQueryBuilder term = (TermQueryBuilder) builder; + String fieldName = term.fieldName(); + String rewrittenFieldName = rewriteFieldName(jobCaps, TermQueryBuilder.NAME, fieldName, null); + return new TermQueryBuilder(rewrittenFieldName, term.value()); + } else if (builder.getWriteableName().equals(TermsQueryBuilder.NAME)) { + TermsQueryBuilder terms = (TermsQueryBuilder) builder; + String fieldName = terms.fieldName(); + String rewrittenFieldName = rewriteFieldName(jobCaps, TermQueryBuilder.NAME, fieldName, null); + return new TermsQueryBuilder(rewrittenFieldName, terms.values()); } else if (builder.getWriteableName().equals(MatchAllQueryBuilder.NAME)) { // no-op return builder; @@ -369,6 +318,64 @@ static QueryBuilder rewriteQuery(QueryBuilder builder, Set jobCap } } + private static String rewriteFieldName(Set jobCaps, + String builderName, + String fieldName, + String timeZone) { + List incompatibleTimeZones = timeZone == null ? Collections.emptyList() : new ArrayList<>(); + List rewrittenFieldNames = jobCaps.stream() + // We only care about job caps that have the query's target field + .filter(caps -> caps.getFieldCaps().keySet().contains(fieldName)) + .map(caps -> { + RollupJobCaps.RollupFieldCaps fieldCaps = caps.getFieldCaps().get(fieldName); + return fieldCaps.getAggs().stream() + // For now, we only allow filtering on grouping fields + .filter(agg -> { + String type = (String)agg.get(RollupField.AGG); + + // If the cap is for a date_histo, and the query is a range, the timezones need to match + if (type.equals(DateHistogramAggregationBuilder.NAME) && timeZone != null) { + boolean matchingTZ = ((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName())) + .equalsIgnoreCase(timeZone); + if (matchingTZ == false) { + incompatibleTimeZones.add((String)agg.get(DateHistoGroupConfig.TIME_ZONE.getPreferredName())); + } + return matchingTZ; + } + // Otherwise just make sure it's one of the three groups + return type.equals(TermsAggregationBuilder.NAME) + || type.equals(DateHistogramAggregationBuilder.NAME) + || type.equals(HistogramAggregationBuilder.NAME); + }) + // Rewrite the field name to our convention (e.g. "foo" -> "date_histogram.foo.timestamp") + .map(agg -> { + if (agg.get(RollupField.AGG).equals(DateHistogramAggregationBuilder.NAME)) { + return RollupField.formatFieldName(fieldName, (String)agg.get(RollupField.AGG), RollupField.TIMESTAMP); + } else { + return RollupField.formatFieldName(fieldName, (String)agg.get(RollupField.AGG), RollupField.VALUE); + } + }) + .collect(Collectors.toList()); + }) + .distinct() + .collect(ArrayList::new, List::addAll, List::addAll); + if (rewrittenFieldNames.isEmpty()) { + if (incompatibleTimeZones.isEmpty()) { + throw new IllegalArgumentException("Field [" + fieldName + "] in [" + builderName + + "] query is not available in selected rollup indices, cannot query."); + } else { + throw new IllegalArgumentException("Field [" + fieldName + "] in [" + builderName + + "] query was found in rollup indices, but requested timezone is not compatible. Options include: " + + incompatibleTimeZones); + } + } else if (rewrittenFieldNames.size() > 1) { + throw new IllegalArgumentException("Ambiguous field name resolution when mapping to rolled fields. Field name [" + + fieldName + "] was mapped to: [" + Strings.collectionToDelimitedString(rewrittenFieldNames, ",") + "]."); + } else { + return rewrittenFieldNames.get(0); + } + } + static RollupSearchContext separateIndices(String[] indices, ImmutableOpenMap indexMetaData) { if (indices.length == 0) { diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java index e37fe3b0e326f..218a1360ae88f 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java @@ -25,9 +25,11 @@ import org.elasticsearch.index.query.DisMaxQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.script.ScriptService; @@ -61,6 +63,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -153,7 +156,7 @@ public void testRangeWrongTZ() { "compatible. Options include: [UTC]")); } - public void testTerms() { + public void testTermQuery() { RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig(); group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build()); @@ -166,6 +169,23 @@ public void testTerms() { assertThat(((TermQueryBuilder)rewritten).fieldName(), equalTo("foo.terms.value")); } + public void testTermsQuery() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig(); + group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build()); + job.setGroupConfig(group.build()); + RollupJobCaps cap = new RollupJobCaps(job.build()); + Set caps = new HashSet<>(); + caps.add(cap); + QueryBuilder original = new TermsQueryBuilder("foo", Arrays.asList("bar", "baz")); + QueryBuilder rewritten = + TransportRollupSearchAction.rewriteQuery(original, caps); + assertThat(rewritten, instanceOf(TermsQueryBuilder.class)); + assertNotSame(rewritten, original); + assertThat(((TermsQueryBuilder)rewritten).fieldName(), equalTo("foo.terms.value")); + assertThat(((TermsQueryBuilder)rewritten).values(), equalTo(Arrays.asList("bar", "baz"))); + } + public void testCompounds() { RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig(); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java index f22a6c87a3ff1..e465c7883cfdd 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java @@ -122,6 +122,37 @@ public void testEmptyIndexPattern() { assertThat(e.getMessage(), equalTo("An index pattern is mandatory.")); } + public void testMatchAllIndexPattern() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + job.setIndexPattern("*"); + Exception e = expectThrows(IllegalArgumentException.class, job::build); + assertThat(e.getMessage(), equalTo("Index pattern must not match all indices (as it would match it's own rollup index")); + } + + public void testMatchOwnRollupPatternPrefix() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + job.setIndexPattern("foo-*"); + job.setRollupIndex("foo-rollup"); + Exception e = expectThrows(IllegalArgumentException.class, job::build); + assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed.")); + } + + public void testMatchOwnRollupPatternSuffix() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + job.setIndexPattern("*-rollup"); + job.setRollupIndex("foo-rollup"); + Exception e = expectThrows(IllegalArgumentException.class, job::build); + assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed.")); + } + + public void testIndexPatternIdenticalToRollup() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + job.setIndexPattern("foo"); + job.setRollupIndex("foo"); + Exception e = expectThrows(IllegalArgumentException.class, job::build); + assertThat(e.getMessage(), equalTo("Rollup index may not be the same as the index pattern.")); + } + public void testEmptyRollupIndex() { RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); job.setRollupIndex(""); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml index 568a6261cda9b..717be0d6b250f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml @@ -188,4 +188,3 @@ setup: ] } -