diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 1a1145a237a76..bd80deda89c6f 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -24,22 +24,22 @@ buildscript { } } dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.2' + classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4' } } apply plugin: 'elasticsearch.build' -// build an uberjar with all benchmarks -apply plugin: 'com.github.johnrengelman.shadow' -// have the shadow plugin provide the runShadow task -apply plugin: 'application' + +// order of this seciont matters, see: https://github.com/johnrengelman/shadow/issues/336 +apply plugin: 'application' // have the shadow plugin provide the runShadow task +mainClassName = 'org.openjdk.jmh.Main' +apply plugin: 'com.github.johnrengelman.shadow' // build an uberjar with all benchmarks // Not published so no need to assemble tasks.remove(assemble) build.dependsOn.remove('assemble') archivesBaseName = 'elasticsearch-benchmarks' -mainClassName = 'org.openjdk.jmh.Main' test.enabled = false diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 1559038aef486..89fe9cf330a35 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ elasticsearch = 7.0.0-alpha1 -lucene = 7.4.0-snapshot-1ed95c097b +lucene = 7.4.0-snapshot-6705632810 # optional dependencies spatial4j = 0.7 diff --git a/client/rest/src/main/java/org/elasticsearch/client/Request.java b/client/rest/src/main/java/org/elasticsearch/client/Request.java index bf0012339fb41..92610239cae92 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Request.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Request.java @@ -19,8 +19,10 @@ package org.elasticsearch.client; +import org.apache.http.entity.ContentType; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.nio.entity.NStringEntity; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; import java.util.Arrays; @@ -103,6 +105,17 @@ public void setEntity(HttpEntity entity) { this.entity = entity; } + /** + * Set the body of the request to a string. If not set or set to + * {@code null} then no body is sent with the request. The + * {@code Content-Type} will be sent as {@code application/json}. + * If you need a different content type then use + * {@link #setEntity(HttpEntity)}. + */ + public void setJsonEntity(String entity) { + setEntity(entity == null ? null : new NStringEntity(entity, ContentType.APPLICATION_JSON)); + } + /** * The body of the request. If {@code null} then no body * is sent with the request. diff --git a/client/rest/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest/src/test/java/org/elasticsearch/client/RequestTests.java index 98fcf8421ae6a..b83115a5341dd 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RequestTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RequestTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.client; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -27,9 +29,11 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; +import org.apache.http.nio.entity.NStringEntity; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; public class RequestTests extends RestClientTestCase { @@ -99,12 +103,27 @@ public void testSetEntity() { final String endpoint = randomAsciiLettersOfLengthBetween(1, 10); final HttpEntity entity = randomBoolean() ? new StringEntity(randomAsciiLettersOfLengthBetween(1, 100), ContentType.TEXT_PLAIN) : null; - Request request = new Request(method, endpoint); + Request request = new Request(method, endpoint); request.setEntity(entity); assertEquals(entity, request.getEntity()); } + public void testSetJsonEntity() throws IOException { + final String method = randomFrom(new String[] {"GET", "PUT", "POST", "HEAD", "DELETE"}); + final String endpoint = randomAsciiLettersOfLengthBetween(1, 10); + + Request request = new Request(method, endpoint); + assertNull(request.getEntity()); + + final String json = randomAsciiLettersOfLengthBetween(1, 100); + request.setJsonEntity(json); + assertEquals(ContentType.APPLICATION_JSON.toString(), request.getEntity().getContentType().getValue()); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + request.getEntity().writeTo(os); + assertEquals(json, new String(os.toByteArray(), ContentType.APPLICATION_JSON.getCharset())); + } + public void testSetHeaders() { final String method = randomFrom(new String[] {"GET", "PUT", "POST", "HEAD", "DELETE"}); final String endpoint = randomAsciiLettersOfLengthBetween(1, 10); diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java index 3f57e8c391215..5979c508de287 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java @@ -35,7 +35,9 @@ import org.apache.http.nio.entity.NStringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.mocksocket.MockHttpServer; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import java.io.IOException; @@ -69,20 +71,20 @@ */ public class RestClientSingleHostIntegTests extends RestClientTestCase { - private static HttpServer httpServer; - private static RestClient restClient; - private static String pathPrefix; - private static Header[] defaultHeaders; + private HttpServer httpServer; + private RestClient restClient; + private String pathPrefix; + private Header[] defaultHeaders; - @BeforeClass - public static void startHttpServer() throws Exception { + @Before + public void startHttpServer() throws Exception { pathPrefix = randomBoolean() ? "/testPathPrefix/" + randomAsciiLettersOfLengthBetween(1, 5) : ""; httpServer = createHttpServer(); defaultHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header-default"); restClient = createRestClient(false, true); } - private static HttpServer createHttpServer() throws Exception { + private HttpServer createHttpServer() throws Exception { HttpServer httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); httpServer.start(); //returns a different status code depending on the path @@ -127,7 +129,7 @@ public void handle(HttpExchange httpExchange) throws IOException { } } - private static RestClient createRestClient(final boolean useAuth, final boolean usePreemptiveAuth) { + private RestClient createRestClient(final boolean useAuth, final boolean usePreemptiveAuth) { // provide the username/password for every request final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("user", "pass")); @@ -155,8 +157,8 @@ public HttpAsyncClientBuilder customizeHttpClient(final HttpAsyncClientBuilder h return restClientBuilder.build(); } - @AfterClass - public static void stopHttpServers() throws IOException { + @After + public void stopHttpServers() throws IOException { restClient.close(); restClient = null; httpServer.stop(0); diff --git a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java index aa89a7d76ab26..5ee97399b34e6 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java +++ b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java @@ -168,10 +168,13 @@ public void onFailure(Exception exception) { request.addParameter("pretty", "true"); //end::rest-client-parameters //tag::rest-client-body - request.setEntity(new StringEntity( + request.setEntity(new NStringEntity( "{\"json\":\"text\"}", ContentType.APPLICATION_JSON)); //end::rest-client-body + //tag::rest-client-body-shorter + request.setJsonEntity("{\"json\":\"text\"}"); + //end::rest-client-body-shorter //tag::rest-client-headers request.setHeaders( new BasicHeader("Accept", "text/plain"), diff --git a/distribution/build.gradle b/distribution/build.gradle index c1ab5b76148b3..266cb8f8b270a 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -312,6 +312,22 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { task run(type: RunTask) { distribution = System.getProperty('run.distribution', 'zip') + if (distribution == 'zip') { + String licenseType = System.getProperty("license_type", "basic") + if (licenseType == 'trial') { + setting 'xpack.ml.enabled', 'true' + setting 'xpack.graph.enabled', 'true' + setting 'xpack.watcher.enabled', 'true' + setting 'xpack.license.self_generated.type', 'trial' + } else if (licenseType != 'basic') { + throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].") + } + setting 'xpack.security.enabled', 'true' + setting 'xpack.monitoring.enabled', 'true' + setting 'xpack.sql.enabled', 'true' + setting 'xpack.rollup.enabled', 'true' + keystoreSetting 'bootstrap.password', 'password' + } } /** diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index b24f694a4bf56..6eb26fde8f9f8 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -115,6 +115,9 @@ Rollup:: * Validate timezone in range queries to ensure they match the selected job when searching ({pull}30338[#30338]) +SQL:: +* Fix parsing of Dates containing milliseconds ({pull}30419[#30419]) + [float] === Regressions Fail snapshot operations early when creating or deleting a snapshot on a repository that has been @@ -166,6 +169,7 @@ copying source settings on index resize operations] ({pull}30255[#30255], {pull} Added new "Request" object flavored request methods in the RestClient. Prefer these instead of the multi-argument versions. ({pull}29623[#29623]) +Added `setJsonEntity` to `Request` object so it is marginally easier to send JSON. ({pull}30447[#30447]) Watcher HTTP client used in watches now allows more parallel connections to the same endpoint and evicts long running connections. ({pull}30130[#30130]) @@ -201,6 +205,8 @@ Rollup:: * Validate timezone in range queries to ensure they match the selected job when searching ({pull}30338[#30338]) +SQL:: +* Fix parsing of Dates containing milliseconds ({pull}30419[#30419]) Allocation:: @@ -241,6 +247,9 @@ Reduce the number of object allocations made by {security} when resolving the in Respect accept header on requests with no handler ({pull}30383[#30383]) +SQL:: +* Fix parsing of Dates containing milliseconds ({pull}30419[#30419]) + //[float] //=== Regressions diff --git a/docs/java-rest/low-level/usage.asciidoc b/docs/java-rest/low-level/usage.asciidoc index 5ffc4332a7681..68367b9a64fdf 100644 --- a/docs/java-rest/low-level/usage.asciidoc +++ b/docs/java-rest/low-level/usage.asciidoc @@ -263,6 +263,14 @@ IMPORTANT: The `ContentType` specified for the `HttpEntity` is important because it will be used to set the `Content-Type` header so that Elasticsearch can properly parse the content. +You can also set it to a `String` which will default to +a `ContentType` of `application/json`. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-body-shorter] +-------------------------------------------------- + And you can set a list of headers to send with the request: ["source","java",subs="attributes,callouts,macros"] diff --git a/docs/reference/indices/forcemerge.asciidoc b/docs/reference/indices/forcemerge.asciidoc index d04ed293168d4..57fe746d59bb2 100644 --- a/docs/reference/indices/forcemerge.asciidoc +++ b/docs/reference/indices/forcemerge.asciidoc @@ -10,6 +10,11 @@ This call will block until the merge is complete. If the http connection is lost, the request will continue in the background, and any new requests will block until the previous force merge is complete. +WARNING: Force merge should only be called against *read-only indices*. Running +force merge against a read-write index can cause very large segments to be produced +(>5Gb per segment), and the merge policy will never consider it for merging again until +it mostly consists of deleted docs. This can cause very large segments to remain in the shards. + [source,js] -------------------------------------------------- POST /twitter/_forcemerge diff --git a/gradle.properties b/gradle.properties index 2511c740bb5b1..08b03629ad53a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ org.gradle.daemon=false -org.gradle.jvmargs=-Xmx1536m +org.gradle.jvmargs=-Xmx2g diff --git a/modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-1ed95c097b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 8303f69b87d7b..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -63ff4af3504881744695f6239fcb3e9c0e3240b1 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-6705632810.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..b333863b94d64 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +f72ad4b6474c2d59b0eed0ca84eddd1f99d29129 \ No newline at end of file diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index a153cc555c81d..765d55dd095c7 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -30,11 +30,17 @@ esplugin { } integTestCluster { + // Modules who's integration is explicitly tested in integration tests + module project(':modules:parent-join') + module project(':modules:lang-painless') // Whitelist reindexing from the local node so we can test reindex-from-remote. setting 'reindex.remote.whitelist', '127.0.0.1:*' } run { + // Modules who's integration is explicitly tested in integration tests + module project(':modules:parent-join') + module project(':modules:lang-painless') // Whitelist reindexing from the local node so we can test reindex-from-remote. setting 'reindex.remote.whitelist', '127.0.0.1:*' } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java index 4dcc0a34758d6..e8e3760882eea 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java @@ -19,10 +19,8 @@ package org.elasticsearch.index.reindex.remote; -import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.lucene.util.BytesRef; +import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; @@ -151,8 +149,7 @@ static Request initialSearch(SearchRequest searchRequest, BytesReference query, } entity.endObject(); - BytesRef bytes = BytesReference.bytes(entity).toBytesRef(); - request.setEntity(new ByteArrayEntity(bytes.bytes, bytes.offset, bytes.length, ContentType.APPLICATION_JSON)); + request.setJsonEntity(Strings.toString(entity)); } catch (IOException e) { throw new ElasticsearchException("unexpected error building entity", e); } @@ -199,7 +196,7 @@ static Request scroll(String scroll, TimeValue keepAlive, Version remoteVersion) if (remoteVersion.before(Version.fromId(2000099))) { // Versions before 2.0.0 extract the plain scroll_id from the body - request.setEntity(new StringEntity(scroll, ContentType.TEXT_PLAIN)); + request.setEntity(new NStringEntity(scroll, ContentType.TEXT_PLAIN)); return request; } @@ -207,7 +204,7 @@ static Request scroll(String scroll, TimeValue keepAlive, Version remoteVersion) entity.startObject() .field("scroll_id", scroll) .endObject(); - request.setEntity(new StringEntity(Strings.toString(entity), ContentType.APPLICATION_JSON)); + request.setJsonEntity(Strings.toString(entity)); } catch (IOException e) { throw new ElasticsearchException("failed to build scroll entity", e); } @@ -219,14 +216,14 @@ static Request clearScroll(String scroll, Version remoteVersion) { if (remoteVersion.before(Version.fromId(2000099))) { // Versions before 2.0.0 extract the plain scroll_id from the body - request.setEntity(new StringEntity(scroll, ContentType.TEXT_PLAIN)); + request.setEntity(new NStringEntity(scroll, ContentType.TEXT_PLAIN)); return request; } try (XContentBuilder entity = JsonXContent.contentBuilder()) { entity.startObject() .array("scroll_id", scroll) .endObject(); - request.setEntity(new StringEntity(Strings.toString(entity), ContentType.APPLICATION_JSON)); + request.setJsonEntity(Strings.toString(entity)); } catch (IOException e) { throw new ElasticsearchException("failed to build clear scroll entity", e); } diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml similarity index 96% rename from qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yml rename to modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml index 9cf022f2de04a..605891d2b32d3 100644 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml @@ -1,5 +1,5 @@ --- -"Response format search failures": +"Response format for search failures": - do: index: index: source diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml similarity index 94% rename from qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml rename to modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml index 8fda091d80da6..617a46dfa66b5 100644 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/10_script.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml @@ -433,3 +433,28 @@ - match: { docs.2._index: index2 } - match: { docs.2._type: type2 } - match: { docs.2._id: fr_789 } + +--- +"Totally broken scripts report the error properly": + - do: + index: + index: twitter + type: tweet + id: 1 + body: { "user": "kimchy" } + - do: + indices.refresh: {} + + - do: + catch: request + reindex: + refresh: true + body: + source: + index: twitter + dest: + index: new_twitter + script: + lang: painless + source: syntax errors are fun! + - match: {error.reason: 'compile error'} diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parent_join.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml similarity index 100% rename from qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/50_reindex_with_parent_join.yml rename to modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml similarity index 95% rename from qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yml rename to modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml index 12801dd3e2ccc..8ace77eee59eb 100644 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml @@ -1,5 +1,5 @@ --- -"Response format search failures": +"Response format for search failures": - do: index: index: source diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml similarity index 95% rename from qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml rename to modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml index dcd5977fb12bc..8ed94347923d1 100644 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml @@ -421,3 +421,25 @@ term: level: 11 - match: { hits.total: 0 } + +--- +"Totally broken scripts report the error properly": + - do: + index: + index: twitter + type: tweet + id: 1 + body: { "user": "kimchy" } + - do: + indices.refresh: {} + + - do: + catch: request + update_by_query: + index: twitter + refresh: true + body: + script: + lang: painless + source: syntax errors are fun! + - match: {error.reason: 'compile error'} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index ab0c271f3ae4f..c8c2c4829d2cf 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -122,8 +122,43 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem Netty4Utils.setup(); } + /* + * Size in bytes of an individual message received by io.netty.handler.codec.MessageAggregator which accumulates the content for an + * HTTP request. This number is used for estimating the maximum number of allowed buffers before the MessageAggregator's internal + * collection of buffers is resized. + * + * By default we assume the Ethernet MTU (1500 bytes) but users can override it with a system property. + */ + private static final ByteSizeValue MTU = new ByteSizeValue(Long.parseLong(System.getProperty("es.net.mtu", "1500"))); + + private static final String SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = "http.netty.max_composite_buffer_components"; + public static Setting SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = - Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope); + new Setting<>(SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS, (s) -> { + ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(s); + /* + * Netty accumulates buffers containing data from all incoming network packets that make up one HTTP request in an instance of + * io.netty.buffer.CompositeByteBuf (think of it as a buffer of buffers). Once its capacity is reached, the buffer will iterate + * over its individual entries and put them into larger buffers (see io.netty.buffer.CompositeByteBuf#consolidateIfNeeded() + * for implementation details). We want to to resize that buffer because this leads to additional garbage on the heap and also + * increases the application's native memory footprint (as direct byte buffers hold their contents off-heap). + * + * With this setting we control the CompositeByteBuf's capacity (which is by default 1024, see + * io.netty.handler.codec.MessageAggregator#DEFAULT_MAX_COMPOSITEBUFFER_COMPONENTS). To determine a proper default capacity for + * that buffer, we need to consider that the upper bound for the size of HTTP requests is determined by `maxContentLength`. The + * number of buffers that are needed depend on how often Netty reads network packets which depends on the network type (MTU). + * We assume here that Elasticsearch receives HTTP requests via an Ethernet connection which has a MTU of 1500 bytes. + * + * Note that we are *not* pre-allocating any memory based on this setting but rather determine the CompositeByteBuf's capacity. + * The tradeoff is between less (but larger) buffers that are contained in the CompositeByteBuf and more (but smaller) buffers. + * With the default max content length of 100MB and a MTU of 1500 bytes we would allow 69905 entries. + */ + long maxBufferComponentsEstimate = Math.round((double) (maxContentLength.getBytes() / MTU.getBytes())); + // clamp value to the allowed range + long maxBufferComponents = Math.max(2, Math.min(maxBufferComponentsEstimate, Integer.MAX_VALUE)); + return String.valueOf(maxBufferComponents); + // Netty's CompositeByteBuf implementation does not allow less than two components. + }, s -> Setting.parseInt(s, 2, Integer.MAX_VALUE, SETTING_KEY_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS), Property.NodeScope); public static final Setting SETTING_HTTP_WORKER_COUNT = new Setting<>("http.netty.worker_count", (s) -> Integer.toString(EsExecutors.numberOfProcessors(s) * 2), @@ -236,8 +271,9 @@ public Netty4HttpServerTransport(Settings settings, NetworkService networkServic this.maxContentLength = maxContentLength; logger.debug("using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}], " + - "receive_predictor[{}], pipelining[{}], pipelining_max_events[{}]", - maxChunkSize, maxHeaderSize, maxInitialLineLength, this.maxContentLength, receivePredictor, pipelining, pipeliningMaxEvents); + "receive_predictor[{}], max_composite_buffer_components[{}], pipelining[{}], pipelining_max_events[{}]", + maxChunkSize, maxHeaderSize, maxInitialLineLength, this.maxContentLength, receivePredictor, maxCompositeBufferComponents, + pipelining, pipeliningMaxEvents); } public Settings settings() { @@ -532,9 +568,7 @@ protected void initChannel(Channel ch) throws Exception { ch.pipeline().addLast("decoder_compress", new HttpContentDecompressor()); ch.pipeline().addLast("encoder", new HttpResponseEncoder()); final HttpObjectAggregator aggregator = new HttpObjectAggregator(Math.toIntExact(transport.maxContentLength.getBytes())); - if (transport.maxCompositeBufferComponents != -1) { - aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); - } + aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); ch.pipeline().addLast("aggregator", aggregator); if (transport.compression) { ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(transport.compressionLevel)); diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-1ed95c097b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index efcbe94ca1857..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5f3c053ef858c58c74a687a40f5451d19b69850b \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-6705632810.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..6720beb8d8682 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +b4e19c53f29fa9b40bd7ad12ff598e3f08d507a3 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-1ed95c097b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 04f81d14df090..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a6e72085f7c2ade43ec0e5f52c227e6f715666ad \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-6705632810.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..861a2110e164a --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +23dd8cb3834f3641d9b3e8bc3d38281389a597bc \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-1ed95c097b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index b10ae670df52a..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a7daed3dc3a67674862002f315cd9193944de783 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-6705632810.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..c7c19feb57df5 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +e8119a17448a6f5512ded0bd2a6faa7fc8e70890 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-1ed95c097b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 76728498136c7..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -25c93466d0a2c41df0cf98de77d632f3f02fa98d \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-6705632810.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..94e8c2698389b --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +336d9ac698066b8cf8a448f193e4a29ef163baa8 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-1ed95c097b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 509f08ed31072..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4688aaa48607ac26f6bf2567052019ab3fb2ff5e \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-6705632810.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..364facee9efe6 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +e1e77951a83fc6a9deab884773314992fefa14f3 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-1ed95c097b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index a6dc434b03ac9..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad71de632c9363c3f200cd5a240686256c7db431 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-6705632810.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..b55d8cf04ec32 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +d4da149a16673c6326f4898ad877756259f676f8 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-1ed95c097b.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 967957ac0ff97..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -96a630a7c4916358f129f6bac8718108811efe1a \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-6705632810.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..dcc2249c45f2f --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +ab4141b43cc6c2680d5f5a0b5086299f38ebec4d \ No newline at end of file diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 8231e15af200c..3c94f4ace7759 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -230,6 +230,11 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { fixtureSupported = true } +boolean legalPath = rootProject.rootDir.toString().contains(" ") == false +if (legalPath == false) { + fixtureSupported = false +} + // Always ignore HA integration tests in the normal integration test runner, they are included below as // part of their own HA-specific integration test tasks. integTestRunner.exclude('**/Ha*TestSuiteIT.class') @@ -248,7 +253,12 @@ if (fixtureSupported) { // Only include the HA integration tests for the HA test task integTestHaRunner.patternSet.setIncludes(['**/Ha*TestSuiteIT.class']) } else { - logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH") + if (legalPath) { + logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH") + } else { + logger.warn("hdfsFixture unsupported since there are spaces in the path: '" + rootProject.rootDir.toString() + "'") + } + // The normal integration test runner will just test that the plugin loads integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic' // HA fixture is unsupported. Don't run them. diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java index fa9cda06589c6..926cf0b2ad4af 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -100,14 +100,9 @@ public InputStream readBlob(String blobName) throws IOException { @Override public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { - if (blobExists(blobName)) { - throw new FileAlreadyExistsException("blob [" + blobName + "] already exists, cannot overwrite"); - } store.execute((Operation) fileContext -> { Path blob = new Path(path, blobName); // we pass CREATE, which means it fails if a blob already exists. - // NOTE: this behavior differs from FSBlobContainer, which passes TRUNCATE_EXISTING - // that should be fixed there, no need to bring truncation into this, give the user an error. EnumSet flags = EnumSet.of(CreateFlag.CREATE, CreateFlag.SYNC_BLOCK); CreateOpts[] opts = {CreateOpts.bufferSize(bufferSize)}; try (FSDataOutputStream stream = fileContext.create(blob, flags, opts)) { @@ -121,6 +116,8 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize) t // if true synchronous behavior is required" stream.hsync(); } + } catch (org.apache.hadoop.fs.FileAlreadyExistsException faee) { + throw new FileAlreadyExistsException(blob.toString(), null, faee.getMessage()); } return null; }); diff --git a/qa/smoke-test-reindex-with-all-modules/build.gradle b/qa/smoke-test-reindex-with-all-modules/build.gradle deleted file mode 100644 index 1d2ef8a325833..0000000000000 --- a/qa/smoke-test-reindex-with-all-modules/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-test' - -integTestCluster { - // Whitelist reindexing from the local node so we can test it. - setting 'reindex.remote.whitelist', '127.0.0.1:*' -} diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java b/qa/smoke-test-reindex-with-all-modules/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java deleted file mode 100644 index db1e62a6b15e6..0000000000000 --- a/qa/smoke-test-reindex-with-all-modules/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.smoketest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; - -public class SmokeTestReindexWithPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - public SmokeTestReindexWithPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); - } -} diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/20_broken.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/20_broken.yml deleted file mode 100644 index fa2c0124ea2a0..0000000000000 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/20_broken.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -"Totally broken scripts report the error properly": - - do: - index: - index: twitter - type: tweet - id: 1 - body: { "user": "kimchy" } - - do: - indices.refresh: {} - - - do: - catch: request - reindex: - refresh: true - body: - source: - index: twitter - dest: - index: new_twitter - script: - lang: painless - source: syntax errors are fun! - - match: {error.reason: 'compile error'} diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/30_timeout.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/30_timeout.yml deleted file mode 100644 index df514c61ceb68..0000000000000 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/reindex/30_timeout.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -"Timeout": - - skip: - version: "all" - reason: painless doesn't support thread.sleep so we need to come up with a better way - - do: - index: - index: twitter - type: tweet - id: 1 - body: { "user": "kimchy" } - - do: - indices.refresh: {} - - - do: - catch: request_timeout - reindex: - refresh: true - body: - source: - index: twitter - timeout: 10 - query: - script: - lang: painless - # Sleep 100x longer than the timeout. That should cause a timeout! - # Return true causes the document to try to be collected which is what actually triggers the timeout. - script: java.lang.Thread.sleep(1000); return true - dest: - index: new_twitter - - is_true: timed_out - - match: {created: 0} - - match: {noops: 0} diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yml deleted file mode 100644 index 8863db5d2aef7..0000000000000 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -"Totally broken scripts report the error properly": - - do: - index: - index: twitter - type: tweet - id: 1 - body: { "user": "kimchy" } - - do: - indices.refresh: {} - - - do: - catch: request - update_by_query: - index: twitter - refresh: true - body: - script: - lang: painless - source: syntax errors are fun! - - match: {error.reason: 'compile error'} diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yml deleted file mode 100644 index ac1ed14a1f894..0000000000000 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -"Timeout": - - skip: - version: "all" - reason: painless doesn't support thread.sleep so we need to come up with a better way - - do: - index: - index: twitter - type: tweet - id: 1 - body: { "user": "kimchy" } - - do: - indices.refresh: {} - - - do: - catch: request_timeout - update_by_query: - index: twitter - refresh: true - search_timeout: 10ms - body: - query: - script: - lang: painless - # Sleep 100x longer than the timeout. That should cause a timeout! - # Return true causes the document to try to be collected which is what actually triggers the timeout. - script: java.lang.Thread.sleep(1000); return true - - is_true: timed_out - - match: {updated: 0} - - match: {noops: 0} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index f64f51b64f021..4cc92177750b6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -107,9 +107,12 @@ setup: --- "Split from 1 to N": - skip: - version: " - 6.99.99" - reason: expects warnings that pre-7.0.0 will not send - features: "warnings" + # when re-enabling uncomment the below skips + version: "all" + reason: "AwaitsFix'ing, see https://github.com/elastic/elasticsearch/issues/30503" + # version: " - 6.99.99" + # reason: expects warnings that pre-7.0.0 will not send + # features: "warnings" - do: indices.create: index: source_one_shard diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml index dc7d83bf978f7..46a889dd863da 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml @@ -1,9 +1,12 @@ --- "Split index ignores target template mapping": - skip: - version: " - 6.99.99" - reason: expects warnings that pre-7.0.0 will not send - features: "warnings" + # when re-enabling uncomment the below skips + version: "all" + reason: "AwaitsFix'ing, see https://github.com/elastic/elasticsearch/issues/30503" + # version: " - 6.99.99" + # reason: expects warnings that pre-7.0.0 will not send + # features: "warnings" # create index - do: diff --git a/server/licenses/lucene-analyzers-common-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-analyzers-common-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index ad3125768abcd..0000000000000 --- a/server/licenses/lucene-analyzers-common-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2b2be48f6622c150496e755497e7bdb8daa46030 \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-analyzers-common-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..ccabc01378088 --- /dev/null +++ b/server/licenses/lucene-analyzers-common-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +f465718b3db829e7660009aac2c1211fd5d74ca0 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-backward-codecs-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 083cc9b1e3911..0000000000000 --- a/server/licenses/lucene-backward-codecs-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6cbafc48e8ac4966377665eb3bbe93f9addf04a5 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-backward-codecs-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..68c4dcebd2ee2 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +d502441e830e1a9d30270442f8e3fd8317fe7bba \ No newline at end of file diff --git a/server/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 20da7d0f78e9d..0000000000000 --- a/server/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0b06e4f6514256a3f187a9892e520638b9c59e63 \ No newline at end of file diff --git a/server/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..c25718d0a9ee0 --- /dev/null +++ b/server/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +5167fb0a14434cb10ec3224e9e32ca668e9f9ad4 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-grouping-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index cb3b6eab5dbb7..0000000000000 --- a/server/licenses/lucene-grouping-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4c71cef87fe513a7a96c2a7980ed6f7c2b015763 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-grouping-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..007ae9bb24eee --- /dev/null +++ b/server/licenses/lucene-grouping-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +488aeecf49413b63a404989ae00b07b20951e76e \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-highlighter-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 5ef97ed311d3c..0000000000000 --- a/server/licenses/lucene-highlighter-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -665e044d1180100940bccd7e8e41dde48e342da3 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-highlighter-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..f1733ffb6826a --- /dev/null +++ b/server/licenses/lucene-highlighter-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +107755edd67cddb3fb9817de50c0bed3a10da19c \ No newline at end of file diff --git a/server/licenses/lucene-join-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-join-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 62afc759b6d8f..0000000000000 --- a/server/licenses/lucene-join-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d343bbf5792f5969288b59b51179acd29d04f4ee \ No newline at end of file diff --git a/server/licenses/lucene-join-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-join-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..71800d6aa153f --- /dev/null +++ b/server/licenses/lucene-join-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +9226fab3b9c6250af52b87061f637c0f8e3114b6 \ No newline at end of file diff --git a/server/licenses/lucene-memory-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-memory-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 1c14266186407..0000000000000 --- a/server/licenses/lucene-memory-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8915f3c93af3348655bcc204289f9011835738a2 \ No newline at end of file diff --git a/server/licenses/lucene-memory-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-memory-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..6cc24bbe98b82 --- /dev/null +++ b/server/licenses/lucene-memory-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +2b7bf384c1933225972f04224d867ec800f5e3a7 \ No newline at end of file diff --git a/server/licenses/lucene-misc-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-misc-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index de39af3dcefe7..0000000000000 --- a/server/licenses/lucene-misc-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7dc67b42eca3b1546a36370b6dcda0f83b2eb7d \ No newline at end of file diff --git a/server/licenses/lucene-misc-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-misc-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..03c146f5c6473 --- /dev/null +++ b/server/licenses/lucene-misc-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +18b770c35db8757dc036b1506870a4ddaad7b1ab \ No newline at end of file diff --git a/server/licenses/lucene-queries-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-queries-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 6d1e73a5abc40..0000000000000 --- a/server/licenses/lucene-queries-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5946d5e2be276f66e9ff6d6111acabb03a9330d9 \ No newline at end of file diff --git a/server/licenses/lucene-queries-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-queries-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..3ecdd79cafd9e --- /dev/null +++ b/server/licenses/lucene-queries-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +683f6436938c67709d0c665c9e1fdef7bd893e4a \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-queryparser-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index ae79d443ff5e7..0000000000000 --- a/server/licenses/lucene-queryparser-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d9fc5fc63f3d861e5af72e11373368e8a4c6bba6 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-queryparser-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..d113267f6f3d1 --- /dev/null +++ b/server/licenses/lucene-queryparser-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +1df20ba64b9aa68f1fa9a15c9ff75f87f94dec47 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-sandbox-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 50022d2e6de46..0000000000000 --- a/server/licenses/lucene-sandbox-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ee283c0a1a717f3e0915de75864a93d043efaee3 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-sandbox-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..7c3391aec27e8 --- /dev/null +++ b/server/licenses/lucene-sandbox-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +895ca714fc62b66ba63d43931730cdc4ef56d35f \ No newline at end of file diff --git a/server/licenses/lucene-spatial-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-spatial-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 4371f75994f29..0000000000000 --- a/server/licenses/lucene-spatial-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e1adf0220a7c052ac81e2919ffac24ac0e5b007c \ No newline at end of file diff --git a/server/licenses/lucene-spatial-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-spatial-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..35d6d5359ebd1 --- /dev/null +++ b/server/licenses/lucene-spatial-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +95ab7e9421bbeb8229d83ac72700b37a521fdf4f \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-spatial-extras-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index a27beb76ff6a4..0000000000000 --- a/server/licenses/lucene-spatial-extras-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6d9306053942c48f43392a634f11a95462b5996e \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-spatial-extras-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..c2f2f39a1f821 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +773ff8c8425d32609ccec6956759ad377dfb8f6b \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-spatial3d-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 8db76c8605c71..0000000000000 --- a/server/licenses/lucene-spatial3d-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2334e8c5f4d0f98659b30e0c2035296e4aae8ff5 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-spatial3d-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..0533067ff0d89 --- /dev/null +++ b/server/licenses/lucene-spatial3d-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +ea711541e243ee768f950041e6e2843d0cc5e695 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-7.4.0-snapshot-1ed95c097b.jar.sha1 b/server/licenses/lucene-suggest-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 84acdbd0d0b01..0000000000000 --- a/server/licenses/lucene-suggest-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f2b2c454eb7b5d73b9df1390ea4730ce3dd4e463 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-7.4.0-snapshot-6705632810.jar.sha1 b/server/licenses/lucene-suggest-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..ce8adccc89a78 --- /dev/null +++ b/server/licenses/lucene-suggest-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +2ca005cf25722ba3777ed93f720f40c937081fa6 \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java index 9f401b6312c46..ebc085ef47e60 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java @@ -54,6 +54,21 @@ public IndicesStatsRequest clear() { return this; } + /** + * Returns the underlying stats flags. + */ + public CommonStatsFlags flags() { + return flags; + } + + /** + * Sets the underlying stats flags. + */ + public IndicesStatsRequest flags(CommonStatsFlags flags) { + this.flags = flags; + return this; + } + /** * Document types to return stats for. Mainly affects {@link #indexing(boolean)} when * enabled, returning specific indexing stats for those types. diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index eeefe793db701..6f2aaa063011f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -99,65 +99,8 @@ protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting sh throw new ShardNotFoundException(indexShard.shardId()); } - CommonStatsFlags flags = new CommonStatsFlags().clear(); - - if (request.docs()) { - flags.set(CommonStatsFlags.Flag.Docs); - } - if (request.store()) { - flags.set(CommonStatsFlags.Flag.Store); - } - if (request.indexing()) { - flags.set(CommonStatsFlags.Flag.Indexing); - flags.types(request.types()); - } - if (request.get()) { - flags.set(CommonStatsFlags.Flag.Get); - } - if (request.search()) { - flags.set(CommonStatsFlags.Flag.Search); - flags.groups(request.groups()); - } - if (request.merge()) { - flags.set(CommonStatsFlags.Flag.Merge); - } - if (request.refresh()) { - flags.set(CommonStatsFlags.Flag.Refresh); - } - if (request.flush()) { - flags.set(CommonStatsFlags.Flag.Flush); - } - if (request.warmer()) { - flags.set(CommonStatsFlags.Flag.Warmer); - } - if (request.queryCache()) { - flags.set(CommonStatsFlags.Flag.QueryCache); - } - if (request.fieldData()) { - flags.set(CommonStatsFlags.Flag.FieldData); - flags.fieldDataFields(request.fieldDataFields()); - } - if (request.segments()) { - flags.set(CommonStatsFlags.Flag.Segments); - flags.includeSegmentFileSizes(request.includeSegmentFileSizes()); - } - if (request.completion()) { - flags.set(CommonStatsFlags.Flag.Completion); - flags.completionDataFields(request.completionFields()); - } - if (request.translog()) { - flags.set(CommonStatsFlags.Flag.Translog); - } - if (request.requestCache()) { - flags.set(CommonStatsFlags.Flag.RequestCache); - } - if (request.recovery()) { - flags.set(CommonStatsFlags.Flag.Recovery); - } - - return new ShardStats( - indexShard.routingEntry(), - indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), indexShard.commitStats(), indexShard.seqNoStats()); + CommonStats commonStats = new CommonStats(indicesService.getIndicesQueryCache(), indexShard, request.flags()); + return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), commonStats, + indexShard.commitStats(), indexShard.seqNoStats()); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java index 1438daf29fd15..780bf037f0e28 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java @@ -75,7 +75,7 @@ public String getExplanation() { @Override public void readFrom(StreamInput in) throws IOException { - if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + if (in.getVersion().onOrAfter(Version.V_6_4_0)) { index = in.readOptionalString(); } else { index = in.readString(); @@ -92,7 +92,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + if (out.getVersion().onOrAfter(Version.V_6_4_0)) { out.writeOptionalString(index); } else { out.writeString(index); diff --git a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java index 20a6602b5c5ad..54a6568af3fa2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java @@ -386,7 +386,7 @@ public Discovery.AckListener createAckListener(ThreadPool threadPool, ClusterSta } }); - return new DelegetingAckListener(ackListeners); + return new DelegatingAckListener(ackListeners); } public boolean clusterStateUnchanged() { @@ -541,11 +541,11 @@ protected void warnAboutSlowTaskIfNeeded(TimeValue executionTime, String source) } } - private static class DelegetingAckListener implements Discovery.AckListener { + private static class DelegatingAckListener implements Discovery.AckListener { private final List listeners; - private DelegetingAckListener(List listeners) { + private DelegatingAckListener(List listeners) { this.listeners = listeners; } @@ -555,11 +555,6 @@ public void onNodeAck(DiscoveryNode node, @Nullable Exception e) { listener.onNodeAck(node, e); } } - - @Override - public void onTimeout() { - throw new UnsupportedOperationException("no timeout delegation"); - } } private static class AckCountDownListener implements Discovery.AckListener { @@ -614,7 +609,6 @@ public void onNodeAck(DiscoveryNode node, @Nullable Exception e) { } } - @Override public void onTimeout() { if (countDown.fastForward()) { logger.trace("timeout waiting for acknowledgement for cluster_state update (version: {})", clusterStateVersion); diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java index f960664306f08..a9600681d1605 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java @@ -123,9 +123,6 @@ public InputStream readBlob(String name) throws IOException { @Override public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { - if (blobExists(blobName)) { - throw new FileAlreadyExistsException("blob [" + blobName + "] already exists, cannot overwrite"); - } final Path file = path.resolve(blobName); try (OutputStream outputStream = Files.newOutputStream(file, StandardOpenOption.CREATE_NEW)) { Streams.copy(inputStream, outputStream); diff --git a/server/src/main/java/org/elasticsearch/common/cache/Cache.java b/server/src/main/java/org/elasticsearch/common/cache/Cache.java index 620612619104b..0db4d718709d8 100644 --- a/server/src/main/java/org/elasticsearch/common/cache/Cache.java +++ b/server/src/main/java/org/elasticsearch/common/cache/Cache.java @@ -206,34 +206,33 @@ private static class CacheSegment { */ Entry get(K key, long now, Predicate> isExpired, Consumer> onExpiration) { CompletableFuture> future; - Entry entry = null; try (ReleasableLock ignored = readLock.acquire()) { future = map.get(key); } if (future != null) { + Entry entry; try { - entry = future.handle((ok, ex) -> { - if (ok != null && !isExpired.test(ok)) { - segmentStats.hit(); - ok.accessTime = now; - return ok; - } else { - segmentStats.miss(); - if (ok != null) { - assert isExpired.test(ok); - onExpiration.accept(ok); - } - return null; - } - }).get(); - } catch (ExecutionException | InterruptedException e) { + entry = future.get(); + } catch (ExecutionException e) { + assert future.isCompletedExceptionally(); + segmentStats.miss(); + return null; + } catch (InterruptedException e) { throw new IllegalStateException(e); } - } - else { + if (isExpired.test(entry)) { + segmentStats.miss(); + onExpiration.accept(entry); + return null; + } else { + segmentStats.hit(); + entry.accessTime = now; + return entry; + } + } else { segmentStats.miss(); + return null; } - return entry; } /** @@ -269,30 +268,18 @@ Tuple, Entry> put(K key, V value, long now) { /** * remove an entry from the segment * - * @param key the key of the entry to remove from the cache - * @return the removed entry if there was one, otherwise null + * @param key the key of the entry to remove from the cache + * @param onRemoval a callback for the removed entry */ - Entry remove(K key) { + void remove(K key, Consumer>> onRemoval) { CompletableFuture> future; - Entry entry = null; try (ReleasableLock ignored = writeLock.acquire()) { future = map.remove(key); } if (future != null) { - try { - entry = future.handle((ok, ex) -> { - if (ok != null) { - segmentStats.eviction(); - return ok; - } else { - return null; - } - }).get(); - } catch (ExecutionException | InterruptedException e) { - throw new IllegalStateException(e); - } + segmentStats.eviction(); + onRemoval.accept(future); } - return entry; } private static class SegmentStats { @@ -476,12 +463,18 @@ private void put(K key, V value, long now) { */ public void invalidate(K key) { CacheSegment segment = getCacheSegment(key); - Entry entry = segment.remove(key); - if (entry != null) { - try (ReleasableLock ignored = lruLock.acquire()) { - delete(entry, RemovalNotification.RemovalReason.INVALIDATED); + segment.remove(key, f -> { + try { + Entry entry = f.get(); + try (ReleasableLock ignored = lruLock.acquire()) { + delete(entry, RemovalNotification.RemovalReason.INVALIDATED); + } + } catch (ExecutionException e) { + // ok + } catch (InterruptedException e) { + throw new IllegalStateException(e); } - } + }); } /** @@ -632,7 +625,7 @@ public void remove() { Entry entry = current; if (entry != null) { CacheSegment segment = getCacheSegment(entry.key); - segment.remove(entry.key); + segment.remove(entry.key, f -> {}); try (ReleasableLock ignored = lruLock.acquire()) { current = null; delete(entry, RemovalNotification.RemovalReason.INVALIDATED); @@ -717,7 +710,7 @@ private void evictEntry(Entry entry) { CacheSegment segment = getCacheSegment(entry.key); if (segment != null) { - segment.remove(entry.key); + segment.remove(entry.key, f -> {}); } delete(entry, RemovalNotification.RemovalReason.EVICTED); } diff --git a/server/src/main/java/org/elasticsearch/discovery/Discovery.java b/server/src/main/java/org/elasticsearch/discovery/Discovery.java index 3842e68d1006b..9c70876032442 100644 --- a/server/src/main/java/org/elasticsearch/discovery/Discovery.java +++ b/server/src/main/java/org/elasticsearch/discovery/Discovery.java @@ -49,7 +49,6 @@ public interface Discovery extends LifecycleComponent { interface AckListener { void onNodeAck(DiscoveryNode node, @Nullable Exception e); - void onTimeout(); } class FailedToCommitClusterStateException extends ElasticsearchException { diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java index 34c29ab0f1890..a07b4186ed594 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java @@ -177,7 +177,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { throw new QueryShardException(context, "illegal latitude value [{}] for [{}]", point.lat(), GeoPolygonQueryBuilder.NAME); } - if (!GeoUtils.isValidLongitude(point.lat())) { + if (!GeoUtils.isValidLongitude(point.lon())) { throw new QueryShardException(context, "illegal longitude value [{}] for [{}]", point.lon(), GeoPolygonQueryBuilder.NAME); } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index ae3a6aada0a7f..fcbc54efbf780 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -950,6 +950,20 @@ protected void finalize(final List snapshots, final BlobStoreIndexShardSnapshots updatedSnapshots = new BlobStoreIndexShardSnapshots(snapshots); try { + // Delete temporary index files first, as we might otherwise fail in the next step creating the new index file if an earlier + // attempt to write an index file with this generation failed mid-way after creating the temporary file. + for (final String blobName : blobs.keySet()) { + if (indexShardSnapshotsFormat.isTempBlobName(blobName)) { + try { + blobContainer.deleteBlobIgnoringIfNotExists(blobName); + } catch (IOException e) { + logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete index blob [{}] during finalization", + snapshotId, shardId, blobName), e); + throw e; + } + } + } + // If we deleted all snapshots, we don't need to create a new index file if (snapshots.size() > 0) { indexShardSnapshotsFormat.writeAtomic(updatedSnapshots, blobContainer, indexGeneration); @@ -957,7 +971,7 @@ protected void finalize(final List snapshots, // Delete old index files for (final String blobName : blobs.keySet()) { - if (indexShardSnapshotsFormat.isTempBlobName(blobName) || blobName.startsWith(SNAPSHOT_INDEX_PREFIX)) { + if (blobName.startsWith(SNAPSHOT_INDEX_PREFIX)) { try { blobContainer.deleteBlobIgnoringIfNotExists(blobName); } catch (IOException e) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java index fd70b7461ec67..715da019e6df4 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.rest.action.admin.indices; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; @@ -57,23 +59,10 @@ public String getName() { static final Map> METRICS; static { - final Map> metrics = new HashMap<>(); - metrics.put("docs", r -> r.docs(true)); - metrics.put("store", r -> r.store(true)); - metrics.put("indexing", r -> r.indexing(true)); - metrics.put("search", r -> r.search(true)); - metrics.put("get", r -> r.get(true)); - metrics.put("merge", r -> r.merge(true)); - metrics.put("refresh", r -> r.refresh(true)); - metrics.put("flush", r -> r.flush(true)); - metrics.put("warmer", r -> r.warmer(true)); - metrics.put("query_cache", r -> r.queryCache(true)); - metrics.put("segments", r -> r.segments(true)); - metrics.put("fielddata", r -> r.fieldData(true)); - metrics.put("completion", r -> r.completion(true)); - metrics.put("request_cache", r -> r.requestCache(true)); - metrics.put("recovery", r -> r.recovery(true)); - metrics.put("translog", r -> r.translog(true)); + Map> metrics = new HashMap<>(); + for (Flag flag : CommonStatsFlags.Flag.values()) { + metrics.put(flag.getRestName(), m -> m.flags().set(flag, true)); + } METRICS = Collections.unmodifiableMap(metrics); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 940b657887319..a7f7ed6f52546 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -80,6 +81,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30416") public class SplitIndexIT extends ESIntegTestCase { @Override @@ -283,7 +285,6 @@ public void assertAllUniqueDocs(SearchResponse response, int numDocs) { assertEquals(numDocs, ids.size()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30432") public void testSplitIndexPrimaryTerm() throws Exception { final List factors = Arrays.asList(1, 2, 4, 8); final List numberOfShardsFactors = randomSubsetOf(scaledRandomIntBetween(1, factors.size()), factors); diff --git a/server/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/server/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java deleted file mode 100644 index cd1a133643339..0000000000000 --- a/server/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.bwcompat; - -import org.elasticsearch.common.io.FileTestUtils; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.repositories.fs.FsRepository; -import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; -import org.elasticsearch.snapshots.SnapshotRestoreException; -import org.elasticsearch.snapshots.mockstore.MockRepository; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.greaterThan; - -/** - * Tests that restoring from a very old snapshot fails appropriately. - */ -@ClusterScope(scope = Scope.TEST) -public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { - - private static Path repoPath; - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(Environment.PATH_REPO_SETTING.getKey(), repoPath) - .build(); - } - - @BeforeClass - public static void repoSetup() throws IOException { - repoPath = createTempDir("repositories"); - } - - public void testRestoreUnsupportedSnapshots() throws Exception { - String repo = "test_repo"; - String snapshot = "test_1"; - List repoVersions = unsupportedRepoVersions(); - assertThat(repoVersions.size(), greaterThan(0)); - for (String version : repoVersions) { - createRepo("unsupportedrepo", version, repo); - assertUnsupportedIndexFailsToRestore(repo, snapshot); - } - } - - private List unsupportedRepoVersions() throws Exception { - return listRepoVersions("unsupportedrepo"); - } - - private List listRepoVersions(String prefix) throws Exception { - List repoVersions = new ArrayList<>(); - Path repoFiles = getBwcIndicesPath(); - try (DirectoryStream stream = Files.newDirectoryStream(repoFiles, prefix + "-*.zip")) { - for (Path entry : stream) { - String fileName = entry.getFileName().toString(); - String version = fileName.substring(prefix.length() + 1); - version = version.substring(0, version.length() - ".zip".length()); - repoVersions.add(version); - } - } - return repoVersions; - } - - private void createRepo(String prefix, String version, String repo) throws Exception { - Path repoFileFromBuild = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip"); - String repoFileName = repoFileFromBuild.getFileName().toString().split(".zip")[0]; - Path fsRepoPath = repoPath.resolve(repoFileName); - FileTestUtils.unzip(repoFileFromBuild, fsRepoPath, null); - logger.info("--> creating repository [{}] for version [{}]", repo, version); - assertAcked(client().admin().cluster().preparePutRepository(repo) - .setType(MockRepository.TYPE).setSettings(Settings.builder() - .put(FsRepository.REPOSITORIES_LOCATION_SETTING.getKey(), fsRepoPath.getParent().relativize(fsRepoPath).resolve("repo").toString()))); - } - - private void assertUnsupportedIndexFailsToRestore(String repo, String snapshot) throws IOException { - logger.info("--> restoring unsupported snapshot"); - try { - client().admin().cluster().prepareRestoreSnapshot(repo, snapshot).setRestoreGlobalState(true).setWaitForCompletion(true).get(); - fail("should have failed to restore - " + repo); - } catch (SnapshotRestoreException ex) { - assertThat(ex.getMessage(), containsString("snapshot does not exist")); - } - } -} - diff --git a/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java b/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java index 1ab38dff7eb7f..fe64fd16af68c 100644 --- a/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java +++ b/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java @@ -344,7 +344,6 @@ protected long now() { assertEquals(numberOfEntries, cache.stats().getEvictions()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30428") public void testComputeIfAbsentDeadlock() throws BrokenBarrierException, InterruptedException { final int numberOfThreads = randomIntBetween(2, 32); final Cache cache = diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java index 42ec72c981007..c8e85382994c7 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java @@ -814,7 +814,6 @@ public AssertingAckListener publishState(PublishClusterStateAction action, Clust public static class AssertingAckListener implements Discovery.AckListener { private final List> errors = new CopyOnWriteArrayList<>(); - private final AtomicBoolean timeoutOccurred = new AtomicBoolean(); private final CountDownLatch countDown; public AssertingAckListener(int nodeCount) { @@ -829,23 +828,12 @@ public void onNodeAck(DiscoveryNode node, @Nullable Exception e) { countDown.countDown(); } - @Override - public void onTimeout() { - timeoutOccurred.set(true); - // Fast forward the counter - no reason to wait here - long currentCount = countDown.getCount(); - for (long i = 0; i < currentCount; i++) { - countDown.countDown(); - } - } - public void await(long timeout, TimeUnit unit) throws InterruptedException { assertThat(awaitErrors(timeout, unit), emptyIterable()); } public List> awaitErrors(long timeout, TimeUnit unit) throws InterruptedException { countDown.await(timeout, unit); - assertFalse(timeoutOccurred.get()); return errors; } diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java index 9698ab18c198b..f209f771ab089 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java @@ -137,6 +137,7 @@ public void tearDown() throws Exception { private static final UnicastHostsProvider EMPTY_HOSTS_PROVIDER = Collections::emptyList; + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/28685") public void testSimplePings() throws IOException, InterruptedException, ExecutionException { // use ephemeral ports final Settings settings = Settings.builder().put("cluster.name", "test").put(TcpTransport.PORT.getKey(), 0).build(); diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java index b5fb281454010..4ca37638a2226 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java @@ -254,4 +254,38 @@ public void testIgnoreUnmapped() throws IOException { QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext())); assertThat(e.getMessage(), containsString("failed to find geo_point field [unmapped]")); } + + public void testPointValidation() throws IOException { + assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); + QueryShardContext context = createShardContext(); + String queryInvalidLat = "{\n" + + " \"geo_polygon\":{\n" + + " \"" + GEO_POINT_FIELD_NAME + "\":{\n" + + " \"points\":[\n" + + " [-70, 140],\n" + + " [-80, 30],\n" + + " [-90, 20]\n" + + " ]\n" + + " }\n" + + " }\n" + + "}\n"; + + QueryShardException e1 = expectThrows(QueryShardException.class, () -> parseQuery(queryInvalidLat).toQuery(context)); + assertThat(e1.getMessage(), containsString("illegal latitude value [140.0] for [geo_polygon]")); + + String queryInvalidLon = "{\n" + + " \"geo_polygon\":{\n" + + " \"" + GEO_POINT_FIELD_NAME + "\":{\n" + + " \"points\":[\n" + + " [-70, 40],\n" + + " [-80, 30],\n" + + " [-190, 20]\n" + + " ]\n" + + " }\n" + + " }\n" + + "}\n"; + + QueryShardException e2 = expectThrows(QueryShardException.class, () -> parseQuery(queryInvalidLon).toQuery(context)); + assertThat(e2.getMessage(), containsString("illegal longitude value [-190.0] for [geo_polygon]")); + } } diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index 27e1c1af2bb83..74106df224851 100644 --- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -255,6 +255,7 @@ private String syncedFlushDescription(ShardsSyncedFlushResult result) { } @TestLogging("_root:DEBUG") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29392") public void testSyncedFlushSkipOutOfSyncReplicas() throws Exception { internalCluster().ensureAtLeastNumDataNodes(between(2, 3)); final int numberOfReplicas = internalCluster().numDataNodes() - 1; @@ -297,6 +298,7 @@ public void testSyncedFlushSkipOutOfSyncReplicas() throws Exception { } @TestLogging("_root:DEBUG") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29392") public void testDoNotRenewSyncedFlushWhenAllSealed() throws Exception { internalCluster().ensureAtLeastNumDataNodes(between(2, 3)); final int numberOfReplicas = internalCluster().numDataNodes() - 1; diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-0.20.6.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-0.20.6.zip deleted file mode 100644 index fab2f20fe62d2..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-0.20.6.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.Beta2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.Beta2.zip deleted file mode 100644 index 020f6f4e59fd8..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.Beta2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.RC1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.RC1.zip deleted file mode 100644 index a84c50759a7d9..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.RC1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.RC2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.RC2.zip deleted file mode 100644 index e5d65ebaf499f..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.RC2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.zip deleted file mode 100644 index 13f778d965b77..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.1.zip deleted file mode 100644 index 76ed278fade3f..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.2.zip deleted file mode 100644 index 762eabf828648..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.3.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.3.zip deleted file mode 100644 index ba79dea3459bf..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.0.3.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.0.zip deleted file mode 100644 index cbf84c717ac2a..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.1.zip deleted file mode 100644 index 00ea0447b2619..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.2.zip deleted file mode 100644 index 18abd996259b2..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.1.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.0.zip deleted file mode 100644 index f5e62a1a94b52..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.1.zip deleted file mode 100644 index 935e71b0c1262..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.2.zip deleted file mode 100644 index d69b22a0887f7..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.3.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.3.zip deleted file mode 100644 index 295f9f758cc39..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.3.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.4.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.4.zip deleted file mode 100644 index e9efc00c97f36..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.2.4.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.0.zip deleted file mode 100644 index 5a59e21930d91..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.1.zip deleted file mode 100644 index 2ae1d7c0ad3e9..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.2.zip deleted file mode 100644 index c67b99717a5ff..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.3.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.3.zip deleted file mode 100644 index 64e3235142dc1..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.3.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.4.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.4.zip deleted file mode 100644 index 55e67445549c4..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.4.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.5.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.5.zip deleted file mode 100644 index 35a5fd4836c3b..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.5.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.6.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.6.zip deleted file mode 100644 index f1eb21ca4b0d6..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.6.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.7.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.7.zip deleted file mode 100644 index 543c13c2ba223..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.7.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.8.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.8.zip deleted file mode 100644 index 93abac386f9aa..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.8.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.9.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.9.zip deleted file mode 100644 index 7dc194df58e2f..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.3.9.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.0.Beta1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.0.Beta1.zip deleted file mode 100644 index 5adf788623f81..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.0.Beta1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.0.zip deleted file mode 100644 index 21f867c85ead3..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.1.zip deleted file mode 100644 index 18166eabbd97b..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.2.zip deleted file mode 100644 index f03625a1ce16b..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.3.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.3.zip deleted file mode 100644 index d78fbb1975bbb..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.3.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.4.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.4.zip deleted file mode 100644 index 1e8944618ea4d..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.4.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.5.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.5.zip deleted file mode 100644 index fcae43982ae11..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.4.5.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.0.zip deleted file mode 100644 index a55e6c4c3537b..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.1.zip deleted file mode 100644 index 7cccbbe25a9ab..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.2.zip deleted file mode 100644 index cee5783e10aa9..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.5.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.0.zip deleted file mode 100644 index 1c31a0240be12..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.1.zip deleted file mode 100644 index 746f3ce613c89..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.2.zip deleted file mode 100644 index de4c5be79289a..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.6.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.0.zip deleted file mode 100644 index 893689b0f4ce8..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.1.zip deleted file mode 100644 index cd5c6c07c9823..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.2.zip deleted file mode 100644 index a0daaac39d442..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.3.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.3.zip deleted file mode 100644 index 12e13dafc5973..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.3.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.4.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.4.zip deleted file mode 100644 index 86be302153bd2..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.4.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.5.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.5.zip deleted file mode 100644 index 46bada25ced85..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-1.7.5.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-beta1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-beta1.zip deleted file mode 100644 index b32f2a48d7478..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-beta1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-beta2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-beta2.zip deleted file mode 100644 index 2b2663a42012e..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-beta2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-rc1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-rc1.zip deleted file mode 100644 index 59b31f5cc3b97..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0-rc1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0.zip deleted file mode 100644 index ae6668be1cf59..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.1.zip deleted file mode 100644 index c675125226c47..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.2.zip deleted file mode 100644 index 2e1062c294b6d..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.0.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.0.zip deleted file mode 100644 index fdaf6321421d6..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.1.zip deleted file mode 100644 index e29cf4016289d..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.2.zip deleted file mode 100644 index f1c371720c5fb..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.1.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.0.zip deleted file mode 100644 index 1501c1942ade6..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.1.zip deleted file mode 100644 index 93e39514c3cd5..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.2.zip deleted file mode 100644 index 5c937a3c6c145..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.2.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.0.zip deleted file mode 100644 index 575232e09df15..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.1.zip deleted file mode 100644 index f0434a446f034..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.2.zip deleted file mode 100644 index c5ca8a3432527..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.3.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.3.zip deleted file mode 100644 index 4310f8e1efb8f..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.3.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.4.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.4.zip deleted file mode 100644 index 6abccc237b67e..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.4.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.5.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.5.zip deleted file mode 100644 index f80ed069180de..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.3.5.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.0.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.0.zip deleted file mode 100644 index 09591e1d7ed1d..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.0.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.1.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.1.zip deleted file mode 100644 index 2c88ecebe3178..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.1.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.2.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.2.zip deleted file mode 100644 index d2f250c8a90e0..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.2.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.3.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.3.zip deleted file mode 100644 index c1ce2d9fbbb56..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.3.zip and /dev/null differ diff --git a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.5.zip b/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.5.zip deleted file mode 100644 index ec1effccc35be..0000000000000 Binary files a/server/src/test/resources/indices/bwc/unsupportedrepo-2.4.5.zip and /dev/null differ diff --git a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_1.json b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_1.json index e079d64eb8fda..94b9fae143a25 100644 --- a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_1.json +++ b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_1.json @@ -1,19 +1,12 @@ { - "filtered": { - "query": { - "match_all": {} - }, - "filter": { - "geo_polygon": { - "location": { - "points": { - "points": [ - [-70, 40], - [-80, 30], - [-90, 20] - ] - } - } + "geo_polygon": { + "location": { + "points": { + "points": [ + [-70, 40], + [-80, 30], + [-90, 20] + ] } } } diff --git a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_2.json b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_2.json index 0955c260727df..a7363452c54bb 100644 --- a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_2.json +++ b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_2.json @@ -1,21 +1,13 @@ { - "filtered": { - "query": { - "match_all": {} - }, - "filter": { - "geo_polygon": { - "location": { - "points": [ - [-70, 40], - [-80, 30], - [-90, 20] - ], - "something_else": { + "geo_polygon": { + "location": { + "points": [ + [-70, 40], + [-80, 30], + [-90, 20] + ], + "something_else": { - } - - } } } } diff --git a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_3.json b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_3.json index 0ac2a7bbb3abc..eef8c1ca074d1 100644 --- a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_3.json +++ b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_3.json @@ -1,12 +1,5 @@ { - "filtered": { - "query": { - "match_all": {} - }, - "filter": { - "geo_polygon": { - "location": ["WRONG"] - } - } + "geo_polygon": { + "location": ["WRONG"] } } diff --git a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_4.json b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_4.json index 51f6ad0037ea6..b2a65825c36f6 100644 --- a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_4.json +++ b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_4.json @@ -1,19 +1,12 @@ { - "filtered": { - "query": { - "match_all": {} + "geo_polygon": { + "location": { + "points": [ + [-70, 40], + [-80, 30], + [-90, 20] + ] }, - "filter": { - "geo_polygon": { - "location": { - "points": [ - [-70, 40], - [-80, 30], - [-90, 20] - ] - }, - "bla": true - } - } + "bla": true } } diff --git a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_5.json b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_5.json index 6f058f551cf60..5287154af42cc 100644 --- a/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_5.json +++ b/server/src/test/resources/org/elasticsearch/index/query/geo_polygon_exception_5.json @@ -1,19 +1,12 @@ { - "filtered": { - "query": { - "match_all": {} + "geo_polygon": { + "location": { + "points": [ + [-70, 40], + [-80, 30], + [-90, 20] + ] }, - "filter": { - "geo_polygon": { - "location": { - "points": [ - [-70, 40], - [-80, 30], - [-90, 20] - ] - }, - "bla": ["array"] - } - } + "bla": ["array"] } } diff --git a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java deleted file mode 100644 index 4128c9d9efc88..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.io; - -import org.elasticsearch.common.Nullable; -import org.junit.Assert; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; - -/** test helper methods for working with files */ -public class FileTestUtils { - - /** - * Check that a file contains a given String - * @param dir root dir for file - * @param filename relative path from root dir to file - * @param expected expected content (if null, we don't expect any file) - */ - public static void assertFileContent(Path dir, String filename, String expected) throws IOException { - Assert.assertThat(Files.exists(dir), is(true)); - Path file = dir.resolve(filename); - if (expected == null) { - Assert.assertThat("file [" + file + "] should not exist.", Files.exists(file), is(false)); - } else { - assertFileExists(file); - String fileContent = new String(Files.readAllBytes(file), java.nio.charset.StandardCharsets.UTF_8); - // trim the string content to prevent different handling on windows vs. unix and CR chars... - Assert.assertThat(fileContent.trim(), equalTo(expected.trim())); - } - } - - /** - * Unzip a zip file to a destination directory. If the zip file does not exist, an IOException is thrown. - * If the destination directory does not exist, it will be created. - * - * @param zip zip file to unzip - * @param destDir directory to unzip the file to - * @param prefixToRemove the (optional) prefix in the zip file path to remove when writing to the destination directory - * @throws IOException if zip file does not exist, or there was an error reading from the zip file or - * writing to the destination directory - */ - public static void unzip(final Path zip, final Path destDir, @Nullable final String prefixToRemove) throws IOException { - if (Files.notExists(zip)) { - throw new IOException("[" + zip + "] zip file must exist"); - } - Files.createDirectories(destDir); - - try (ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) { - ZipEntry entry; - while ((entry = zipInput.getNextEntry()) != null) { - final String entryPath; - if (prefixToRemove != null) { - if (entry.getName().startsWith(prefixToRemove)) { - entryPath = entry.getName().substring(prefixToRemove.length()); - } else { - throw new IOException("prefix not found: " + prefixToRemove); - } - } else { - entryPath = entry.getName(); - } - final Path path = Paths.get(destDir.toString(), entryPath); - if (entry.isDirectory()) { - Files.createDirectories(path); - } else { - Files.copy(zipInput, path); - } - zipInput.closeEntry(); - } - } - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java index 8aff12edc8a53..743be6d1bcb01 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.NoSuchFileException; import java.util.Arrays; import java.util.HashMap; @@ -149,7 +150,7 @@ public void testVerifyOverwriteFails() throws IOException { final BytesArray bytesArray = new BytesArray(data); writeBlob(container, blobName, bytesArray); // should not be able to overwrite existing blob - expectThrows(IOException.class, () -> writeBlob(container, blobName, bytesArray)); + expectThrows(FileAlreadyExistsException.class, () -> writeBlob(container, blobName, bytesArray)); container.deleteBlob(blobName); writeBlob(container, blobName, bytesArray); // after deleting the previous blob, we should be able to write to it again } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index dfd3713333543..0a51fbdb8bd9b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -519,6 +519,19 @@ public static byte randomByte() { return (byte) random().nextInt(); } + /** + * Helper method to create a byte array of a given length populated with random byte values + * + * @see #randomByte() + */ + public static byte[] randomByteArrayOfLength(int size) { + byte[] bytes = new byte[size]; + for (int i = 0; i < size; i++) { + bytes[i] = randomByte(); + } + return bytes; + } + public static short randomShort() { return (short) random().nextInt(); } diff --git a/x-pack/build.gradle b/x-pack/build.gradle index 88e09d82168f7..91652b9e15015 100644 --- a/x-pack/build.gradle +++ b/x-pack/build.gradle @@ -55,13 +55,4 @@ subprojects { ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-security:${version}": xpackModule('security')] ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-upgrade:${version}": xpackModule('upgrade')] ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-watcher:${version}": xpackModule('watcher')] - - bwcVersions.snapshotProjectNames.each { snapshotName -> - Version snapshot = bwcVersions.getSnapshotForProject(snapshotName) - if (snapshot != null && snapshot.onOrAfter("6.3.0")) { - String snapshotProject = ":x-pack:plugin:bwc:${snapshotName}" - project(snapshotProject).ext.bwcVersion = snapshot - ext.projectSubstitutions["org.elasticsearch.plugin:x-pack:${snapshot}"] = snapshotProject - } - } } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 110b26182ad8e..e4dc314eb72a7 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -158,21 +158,3 @@ integTestCluster { return tmpFile.exists() } } - -run { - def licenseType = System.getProperty("license_type", "basic") - if (licenseType == 'trial') { - setting 'xpack.ml.enabled', 'true' - setting 'xpack.graph.enabled', 'true' - setting 'xpack.watcher.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - } else if (licenseType != 'basic') { - throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "]. Must be " + - "[basic] or [trial].") - } - setting 'xpack.security.enabled', 'true' - setting 'xpack.monitoring.enabled', 'true' - setting 'xpack.sql.enabled', 'true' - setting 'xpack.rollup.enabled', 'true' - keystoreSetting 'bootstrap.password', 'password' -} diff --git a/x-pack/plugin/bwc/build.gradle b/x-pack/plugin/bwc/build.gradle deleted file mode 100644 index 757448e35cd12..0000000000000 --- a/x-pack/plugin/bwc/build.gradle +++ /dev/null @@ -1,226 +0,0 @@ -import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.LoggedExec -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.test.NodeInfo - -import static org.elasticsearch.gradle.BuildPlugin.getJavaHome - -/** - * Subdirectories of this project are dummy projects which does a local - * checkout of the appropriate version's branch, and builds a snapshot. This - * allows backcompat tests to test against the next unreleased versions - * without relying on snapshots. - */ - -subprojects { - - Version bwcVersion = bwcVersions.getSnapshotForProject(project.name) - if (bwcVersion == null) { - // this project wont do anything - return - } - - String bwcBranch - if (project.name == 'next-minor-snapshot') { - // this is always a .x series - bwcBranch = "${bwcVersion.major}.x" - } else { - bwcBranch = "${bwcVersion.major}.${bwcVersion.minor}" - } - - apply plugin: 'distribution' - // Not published so no need to assemble - tasks.remove(assemble) - build.dependsOn.remove('assemble') - - File esCheckoutDir = file("${buildDir}/bwc/checkout-es-${bwcBranch}") - /* Delay building the path as the path will not exist during configuration which will - * fail on Windows due to getting the short name requiring the path to already exist. - */ - Object esCheckoutPath = """${-> - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - esCheckoutDir.mkdirs() - NodeInfo.getShortPathName(esCheckoutDir.toString()) - } else { - esCheckoutDir.toString() - } - }""" - File xpackCheckoutDir = file("${esCheckoutDir}-extra/x-pack-elasticsearch") - Object xpackCheckoutPath = """${-> - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - xpackCheckoutDir.mkdirs() - NodeInfo.getShortPathName(xpackCheckoutDir.toString()) - } else { - xpackCheckoutDir.toString() - } - }""" - - final String remote = System.getProperty("tests.bwc.remote", "elastic") - - task createElasticsearchClone(type: LoggedExec) { - onlyIf { esCheckoutDir.exists() == false } - commandLine = ['git', 'clone', rootDir, esCheckoutPath] - } - - task createXPackClone(type: LoggedExec) { - onlyIf { xpackCheckoutDir.exists() == false } - commandLine = ['git', 'clone', xpackRootProject.projectDir, xpackCheckoutPath] - } - - // we use regular Exec here to ensure we always get output, regardless of logging level - task findElasticsearchRemote(type: Exec) { - dependsOn createElasticsearchClone - workingDir = esCheckoutDir - commandLine = ['git', 'remote', '-v'] - ignoreExitValue = true - ByteArrayOutputStream output = new ByteArrayOutputStream() - standardOutput = output - doLast { - if (execResult.exitValue != 0) { - output.toString('UTF-8').eachLine { line -> logger.error(line) } - execResult.assertNormalExitValue() - } - project.ext.esRemoteExists = false - output.toString('UTF-8').eachLine { - if (it.contains("${remote}\t")) { - project.ext.esRemoteExists = true - } - } - } - } - - task findXPackRemote(type: Exec) { - dependsOn createXPackClone - workingDir = xpackCheckoutDir - commandLine = ['git', 'remote', '-v'] - ignoreExitValue = true - ByteArrayOutputStream output = new ByteArrayOutputStream() - standardOutput = output - doLast { - if (execResult.exitValue != 0) { - output.toString('UTF-8').eachLine { line -> logger.error(line) } - execResult.assertNormalExitValue() - } - project.ext.xpackRemoteExists = false - output.toString('UTF-8').eachLine { - if (it.contains("${remote}\t")) { - project.ext.xpackRemoteExists = true - } - } - } - } - - task addElasticsearchRemote(type: LoggedExec) { - dependsOn findElasticsearchRemote - onlyIf { project.ext.esRemoteExists == false } - workingDir = esCheckoutDir - commandLine = ['git', 'remote', 'add', "${remote}", "git@github.com:${remote}/elasticsearch.git"] - } - - task addXPackRemote(type: LoggedExec) { - dependsOn findXPackRemote - onlyIf { project.ext.xpackRemoteExists == false } - workingDir = xpackCheckoutDir - commandLine = ['git', 'remote', 'add', "${remote}", "git@github.com:${remote}/x-pack-elasticsearch.git"] - } - - task fetchElasticsearchLatest(type: LoggedExec) { - dependsOn addElasticsearchRemote - workingDir = esCheckoutDir - commandLine = ['git', 'fetch', '--all'] - } - - task fetchXPackLatest(type: LoggedExec) { - dependsOn addXPackRemote - workingDir = xpackCheckoutDir - commandLine = ['git', 'fetch', '--all'] - } - - String esBuildMetadataKey = "bwc_refspec_${project.path.substring(1)}_elasticsearch" - task checkoutElasticsearchBwcBranch(type: LoggedExec) { - dependsOn fetchElasticsearchLatest - def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(esBuildMetadataKey, "${remote}/${bwcBranch}")) - workingDir = esCheckoutDir - commandLine = ['git', 'checkout', refspec] - } - - String xpackBuildMetadataKey = "bwc_refspec_${project.path.substring(1)}_xpack" - task checkoutXPackBwcBranch(type: LoggedExec) { - dependsOn fetchXPackLatest - def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(xpackBuildMetadataKey, "${remote}/${bwcBranch}")) - workingDir = xpackCheckoutDir - commandLine = ['git', 'checkout', refspec] - } - - File esBuildMetadataFile = project.file("build/${project.name}_elasticsearch/build_metadata") - task writeElasticsearchBuildMetadata(type: LoggedExec) { - dependsOn checkoutElasticsearchBwcBranch - workingDir = esCheckoutDir - commandLine = ['git', 'rev-parse', 'HEAD'] - ignoreExitValue = true - ByteArrayOutputStream output = new ByteArrayOutputStream() - standardOutput = output - doLast { - if (execResult.exitValue != 0) { - output.toString('UTF-8').eachLine { line -> logger.error(line) } - execResult.assertNormalExitValue() - } - project.mkdir(esBuildMetadataFile.parent) - esBuildMetadataFile.setText("${esBuildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8') - } - } - - File xpackBuildMetadataFile = project.file("build/${project.name}_xpack/build_metadata") - task writeXPackBuildMetadata(type: LoggedExec) { - dependsOn checkoutXPackBwcBranch - workingDir = xpackCheckoutDir - commandLine = ['git', 'rev-parse', 'HEAD'] - ignoreExitValue = true - ByteArrayOutputStream output = new ByteArrayOutputStream() - standardOutput = output - doLast { - if (execResult.exitValue != 0) { - output.toString('UTF-8').eachLine { line -> logger.error(line) } - execResult.assertNormalExitValue() - } - project.mkdir(xpackBuildMetadataFile.parent) - xpackBuildMetadataFile.setText("${xpackBuildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8') - } - } - - File bwcZip = file("${xpackCheckoutDir}/plugin/build/distributions/x-pack-${bwcVersion}.zip") - task buildBwcVersion(type: Exec) { - dependsOn checkoutXPackBwcBranch, checkoutElasticsearchBwcBranch, writeElasticsearchBuildMetadata, writeXPackBuildMetadata - workingDir = xpackCheckoutDir - if (["5.6", "6.0", "6.1"].contains(bwcBranch)) { - // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds - environment('JAVA_HOME', getJavaHome(it, 8)) - } else if ("6.2".equals(bwcBranch)) { - environment('JAVA_HOME', getJavaHome(it, 9)) - } else { - environment('JAVA_HOME', project.compilerJavaHome) - } - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - executable 'cmd' - args '/C', 'call', new File(xpackCheckoutDir, 'gradlew').toString() - } else { - executable new File(xpackCheckoutDir, 'gradlew').toString() - } - args ":x-pack-elasticsearch:plugin:assemble", "-Dbuild.snapshot=true" - final LogLevel logLevel = gradle.startParameter.logLevel - if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) { - args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}" - } - final String showStacktraceName = gradle.startParameter.showStacktrace.name() - assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName) - if (showStacktraceName.equals("ALWAYS")) { - args "--stacktrace" - } else if (showStacktraceName.equals("ALWAYS_FULL")) { - args "--full-stacktrace" - } - } - - artifacts { - 'default' file: bwcZip, name: 'x-pack', type: 'zip', builtBy: buildBwcVersion - } -} diff --git a/x-pack/plugin/bwc/maintenance-bugfix-snapshot/build.gradle b/x-pack/plugin/bwc/maintenance-bugfix-snapshot/build.gradle deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/bwc/next-bugfix-snapshot/build.gradle b/x-pack/plugin/bwc/next-bugfix-snapshot/build.gradle deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/bwc/next-minor-snapshot/build.gradle b/x-pack/plugin/bwc/next-minor-snapshot/build.gradle deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/bwc/staged-minor-snapshot/build.gradle b/x-pack/plugin/bwc/staged-minor-snapshot/build.gradle deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java index f7998a52d496a..1fb387b0b6c2a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java @@ -45,7 +45,7 @@ public PutJobAction.Response newResponse() { public static class Request extends AcknowledgedRequest implements ToXContentObject { public static UpdateJobAction.Request parseRequest(String jobId, XContentParser parser) { - JobUpdate update = JobUpdate.PARSER.apply(parser, null).setJobId(jobId).build(); + JobUpdate update = JobUpdate.EXTERNAL_PARSER.apply(parser, null).setJobId(jobId).build(); return new UpdateJobAction.Request(jobId, update); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 8644254b92162..2c7ee538485b9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -30,26 +30,35 @@ public class JobUpdate implements Writeable, ToXContentObject { public static final ParseField DETECTORS = new ParseField("detectors"); - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + // For internal updates + static final ConstructingObjectParser INTERNAL_PARSER = new ConstructingObjectParser<>( + "job_update", args -> new Builder((String) args[0])); + + // For parsing REST requests + public static final ConstructingObjectParser EXTERNAL_PARSER = new ConstructingObjectParser<>( "job_update", args -> new Builder((String) args[0])); static { - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), Job.ID); - PARSER.declareStringArray(Builder::setGroups, Job.GROUPS); - PARSER.declareStringOrNull(Builder::setDescription, Job.DESCRIPTION); - PARSER.declareObjectArray(Builder::setDetectorUpdates, DetectorUpdate.PARSER, DETECTORS); - PARSER.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.CONFIG_PARSER, Job.MODEL_PLOT_CONFIG); - PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.CONFIG_PARSER, Job.ANALYSIS_LIMITS); - PARSER.declareString((builder, val) -> builder.setBackgroundPersistInterval( - TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL); - PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); - PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); - PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); - PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); - PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT); - PARSER.declareString(Builder::setModelSnapshotId, Job.MODEL_SNAPSHOT_ID); - PARSER.declareString(Builder::setModelSnapshotMinVersion, Job.MODEL_SNAPSHOT_MIN_VERSION); - PARSER.declareLong(Builder::setEstablishedModelMemory, Job.ESTABLISHED_MODEL_MEMORY); + for (ConstructingObjectParser parser : Arrays.asList(INTERNAL_PARSER, EXTERNAL_PARSER)) { + parser.declareString(ConstructingObjectParser.optionalConstructorArg(), Job.ID); + parser.declareStringArray(Builder::setGroups, Job.GROUPS); + parser.declareStringOrNull(Builder::setDescription, Job.DESCRIPTION); + parser.declareObjectArray(Builder::setDetectorUpdates, DetectorUpdate.PARSER, DETECTORS); + parser.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.CONFIG_PARSER, Job.MODEL_PLOT_CONFIG); + parser.declareObject(Builder::setAnalysisLimits, AnalysisLimits.CONFIG_PARSER, Job.ANALYSIS_LIMITS); + parser.declareString((builder, val) -> builder.setBackgroundPersistInterval( + TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL); + parser.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); + parser.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); + parser.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); + parser.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); + parser.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT); + } + // These fields should not be set by a REST request + INTERNAL_PARSER.declareString(Builder::setModelSnapshotId, Job.MODEL_SNAPSHOT_ID); + INTERNAL_PARSER.declareLong(Builder::setEstablishedModelMemory, Job.ESTABLISHED_MODEL_MEMORY); + INTERNAL_PARSER.declareString(Builder::setModelSnapshotMinVersion, Job.MODEL_SNAPSHOT_MIN_VERSION); + INTERNAL_PARSER.declareString(Builder::setJobVersion, Job.JOB_VERSION); } private final String jobId; @@ -67,6 +76,7 @@ public class JobUpdate implements Writeable, ToXContentObject { private final String modelSnapshotId; private final Version modelSnapshotMinVersion; private final Long establishedModelMemory; + private final Version jobVersion; private JobUpdate(String jobId, @Nullable List groups, @Nullable String description, @Nullable List detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig, @@ -74,7 +84,8 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String @Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays, @Nullable Long modelSnapshotRetentionDays, @Nullable List categorisationFilters, @Nullable Map customSettings, @Nullable String modelSnapshotId, - @Nullable Version modelSnapshotMinVersion, @Nullable Long establishedModelMemory) { + @Nullable Version modelSnapshotMinVersion, @Nullable Long establishedModelMemory, + @Nullable Version jobVersion) { this.jobId = jobId; this.groups = groups; this.description = description; @@ -90,6 +101,7 @@ private JobUpdate(String jobId, @Nullable List groups, @Nullable String this.modelSnapshotId = modelSnapshotId; this.modelSnapshotMinVersion = modelSnapshotMinVersion; this.establishedModelMemory = establishedModelMemory; + this.jobVersion = jobVersion; } public JobUpdate(StreamInput in) throws IOException { @@ -119,16 +131,21 @@ public JobUpdate(StreamInput in) throws IOException { } customSettings = in.readMap(); modelSnapshotId = in.readOptionalString(); - if (in.getVersion().onOrAfter(Version.V_6_3_0) && in.readBoolean()) { - modelSnapshotMinVersion = Version.readVersion(in); - } else { - modelSnapshotMinVersion = null; - } if (in.getVersion().onOrAfter(Version.V_6_1_0)) { establishedModelMemory = in.readOptionalLong(); } else { establishedModelMemory = null; } + if (in.getVersion().onOrAfter(Version.V_6_3_0) && in.readBoolean()) { + jobVersion = Version.readVersion(in); + } else { + jobVersion = null; + } + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1) && in.readBoolean()) { + modelSnapshotMinVersion = Version.readVersion(in); + } else { + modelSnapshotMinVersion = null; + } } @Override @@ -155,7 +172,18 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeMap(customSettings); out.writeOptionalString(modelSnapshotId); + if (out.getVersion().onOrAfter(Version.V_6_1_0)) { + out.writeOptionalLong(establishedModelMemory); + } if (out.getVersion().onOrAfter(Version.V_6_3_0)) { + if (jobVersion != null) { + out.writeBoolean(true); + Version.writeVersion(jobVersion, out); + } else { + out.writeBoolean(false); + } + } + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { if (modelSnapshotMinVersion != null) { out.writeBoolean(true); Version.writeVersion(modelSnapshotMinVersion, out); @@ -163,9 +191,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } - if (out.getVersion().onOrAfter(Version.V_6_1_0)) { - out.writeOptionalLong(establishedModelMemory); - } } public String getJobId() { @@ -228,6 +253,10 @@ public Long getEstablishedModelMemory() { return establishedModelMemory; } + public Version getJobVersion() { + return jobVersion; + } + public boolean isAutodetectProcessUpdate() { return modelPlotConfig != null || detectorUpdates != null; } @@ -278,6 +307,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (establishedModelMemory != null) { builder.field(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory); } + if (jobVersion != null) { + builder.field(Job.JOB_VERSION.getPreferredName(), jobVersion); + } builder.endObject(); return builder; } @@ -326,13 +358,16 @@ public Set getUpdateFields() { if (establishedModelMemory != null) { updateFields.add(Job.ESTABLISHED_MODEL_MEMORY.getPreferredName()); } + if (jobVersion != null) { + updateFields.add(Job.JOB_VERSION.getPreferredName()); + } return updateFields; } /** * Updates {@code source} with the new values in this object returning a new {@link Job}. * - * @param source Source job to be updated + * @param source Source job to be updated * @param maxModelMemoryLimit The maximum model memory allowed * @return A new job equivalent to {@code source} updated. */ @@ -408,6 +443,9 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { builder.setEstablishedModelMemory(null); } } + if (jobVersion != null) { + builder.setJobVersion(jobVersion); + } return builder.build(); } @@ -437,14 +475,15 @@ public boolean equals(Object other) { && Objects.equals(this.customSettings, that.customSettings) && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) && Objects.equals(this.modelSnapshotMinVersion, that.modelSnapshotMinVersion) - && Objects.equals(this.establishedModelMemory, that.establishedModelMemory); + && Objects.equals(this.establishedModelMemory, that.establishedModelMemory) + && Objects.equals(this.jobVersion, that.jobVersion); } @Override public int hashCode() { return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings, - modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory); + modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory, jobVersion); } public static class DetectorUpdate implements Writeable, ToXContentObject { @@ -555,6 +594,7 @@ public static class Builder { private String modelSnapshotId; private Version modelSnapshotMinVersion; private Long establishedModelMemory; + private Version jobVersion; public Builder(String jobId) { this.jobId = jobId; @@ -640,10 +680,20 @@ public Builder setEstablishedModelMemory(Long establishedModelMemory) { return this; } + public Builder setJobVersion(Version version) { + this.jobVersion = version; + return this; + } + + public Builder setJobVersion(String version) { + this.jobVersion = Version.fromString(version); + return this; + } + public JobUpdate build() { return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval, renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings, - modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory); + modelSnapshotId, modelSnapshotMinVersion, establishedModelMemory, jobVersion); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityLifecycleServiceField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityLifecycleServiceField.java deleted file mode 100644 index 8e3cc23d8032f..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityLifecycleServiceField.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.security; - -public final class SecurityLifecycleServiceField { - public static final String SECURITY_TEMPLATE_NAME = "security-index-template"; - - private SecurityLifecycleServiceField() {} -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetaData.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetaData.java index 3b8ea2910d13d..6bd6228f2efe1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetaData.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetaData.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -74,13 +75,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; TokenMetaData that = (TokenMetaData)o; - return keys.equals(that.keys) && currentKeyHash.equals(that.currentKeyHash); + return keys.equals(that.keys) && Arrays.equals(currentKeyHash, that.currentKeyHash); } @Override public int hashCode() { int result = keys.hashCode(); - result = 31 * result + currentKeyHash.hashCode(); + result = 31 * result + Arrays.hashCode(currentKeyHash); return result; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 7a976c89cdb40..3663ff14e6302 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -26,6 +26,8 @@ public class JobUpdateTests extends AbstractSerializingTestCase { + private boolean useInternalParser = randomBoolean(); + @Override protected JobUpdate createTestInstance() { JobUpdate.Builder update = new JobUpdate.Builder(randomAlphaOfLength(4)); @@ -84,15 +86,18 @@ protected JobUpdate createTestInstance() { if (randomBoolean()) { update.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10))); } - if (randomBoolean()) { + if (useInternalParser && randomBoolean()) { update.setModelSnapshotId(randomAlphaOfLength(10)); } - if (randomBoolean()) { + if (useInternalParser && randomBoolean()) { update.setModelSnapshotMinVersion(Version.CURRENT); } - if (randomBoolean()) { + if (useInternalParser && randomBoolean()) { update.setEstablishedModelMemory(randomNonNegativeLong()); } + if (useInternalParser && randomBoolean()) { + update.setJobVersion(randomFrom(Version.CURRENT, Version.V_6_2_0, Version.V_6_1_0)); + } return update.build(); } @@ -104,7 +109,11 @@ protected Writeable.Reader instanceReader() { @Override protected JobUpdate doParseInstance(XContentParser parser) { - return JobUpdate.PARSER.apply(parser, null).build(); + if (useInternalParser) { + return JobUpdate.INTERNAL_PARSER.apply(parser, null).build(); + } else { + return JobUpdate.EXTERNAL_PARSER.apply(parser, null).build(); + } } public void testMergeWithJob() { @@ -137,6 +146,7 @@ public void testMergeWithJob() { updateBuilder.setCategorizationFilters(categorizationFilters); updateBuilder.setCustomSettings(customSettings); updateBuilder.setModelSnapshotId(randomAlphaOfLength(10)); + updateBuilder.setJobVersion(Version.V_6_1_0); JobUpdate update = updateBuilder.build(); Job.Builder jobBuilder = new Job.Builder("foo"); @@ -164,6 +174,7 @@ public void testMergeWithJob() { assertEquals(update.getCategorizationFilters(), updatedJob.getAnalysisConfig().getCategorizationFilters()); assertEquals(update.getCustomSettings(), updatedJob.getCustomSettings()); assertEquals(update.getModelSnapshotId(), updatedJob.getModelSnapshotId()); + assertEquals(update.getJobVersion(), updatedJob.getJobVersion()); for (JobUpdate.DetectorUpdate detectorUpdate : update.getDetectorUpdates()) { assertNotNull(updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription()); assertEquals(detectorUpdate.getDescription(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/TokenMetaDataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/TokenMetaDataTests.java new file mode 100644 index 0000000000000..77f7c4dd3ad04 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/TokenMetaDataTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.security.authc; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class TokenMetaDataTests extends ESTestCase { + + public void testEqualsAndHashCode() { + final int numKeyAndTimestamps = scaledRandomIntBetween(1, 8); + final List keyAndTimestampList = generateKeyAndTimestampListOfSize(numKeyAndTimestamps); + final byte[] currentKeyHash = randomByteArrayOfLength(8); + final TokenMetaData original = new TokenMetaData(keyAndTimestampList, currentKeyHash); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(original, tokenMetaData -> { + final List copiedList = new ArrayList<>(keyAndTimestampList); + final byte[] copyKeyHash = Arrays.copyOf(currentKeyHash, currentKeyHash.length); + return new TokenMetaData(copiedList, copyKeyHash); + }, tokenMetaData -> { + final List modifiedList = generateKeyAndTimestampListOfSize(numKeyAndTimestamps); + return new TokenMetaData(modifiedList, currentKeyHash); + }); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(original, tokenMetaData -> { + BytesStreamOutput out = new BytesStreamOutput(); + tokenMetaData.writeTo(out); + return new TokenMetaData(out.bytes().streamInput()); + }, tokenMetaData -> { + final byte[] modifiedKeyHash = randomByteArrayOfLength(8); + return new TokenMetaData(keyAndTimestampList, modifiedKeyHash); + }); + } + + private List generateKeyAndTimestampListOfSize(int size) { + final List keyAndTimestampList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + keyAndTimestampList.add( + new KeyAndTimestamp(new SecureString(randomAlphaOfLengthBetween(1, 12).toCharArray()), randomNonNegativeLong())); + } + return keyAndTimestampList; + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index a448cd9c65c17..4e4f86baec768 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -231,7 +231,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; -import static org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; @@ -424,8 +424,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(realms); components.add(reservedRealm); - securityLifecycleService.securityIndex().addIndexHealthChangeListener(nativeRoleMappingStore::onSecurityIndexHealthChange); - securityLifecycleService.securityIndex().addIndexOutOfDateListener(nativeRoleMappingStore::onSecurityIndexOutOfDateChange); + securityLifecycleService.securityIndex().addIndexStateListener(nativeRoleMappingStore::onSecurityIndexStateChange); AuthenticationFailureHandler failureHandler = null; String extensionName = null; @@ -458,8 +457,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste } final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore, rolesProviders, threadPool.getThreadContext(), getLicenseState()); - securityLifecycleService.securityIndex().addIndexHealthChangeListener(allRolesStore::onSecurityIndexHealthChange); - securityLifecycleService.securityIndex().addIndexOutOfDateListener(allRolesStore::onSecurityIndexOutOfDateChange); + securityLifecycleService.securityIndex().addIndexStateListener(allRolesStore::onSecurityIndexStateChange); // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be // minimal getLicenseState().addListener(allRolesStore::invalidateAll); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java index 099c9cc625b14..9d05ec3b71d6b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java @@ -49,8 +49,6 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust public static final String INTERNAL_SECURITY_INDEX = SecurityIndexManager.INTERNAL_SECURITY_INDEX; public static final String SECURITY_INDEX_NAME = ".security"; - private static final Version MIN_READ_VERSION = Version.V_5_0_0; - private final Settings settings; private final ThreadPool threadPool; private final IndexAuditTrail indexAuditTrail; @@ -125,36 +123,7 @@ private void close() { } } - public static boolean securityIndexMappingSufficientToRead(ClusterState clusterState, Logger logger) { - return checkMappingVersions(clusterState, logger, MIN_READ_VERSION::onOrBefore); - } - - static boolean securityIndexMappingUpToDate(ClusterState clusterState, Logger logger) { - return checkMappingVersions(clusterState, logger, Version.CURRENT::equals); - } - - private static boolean checkMappingVersions(ClusterState clusterState, Logger logger, Predicate versionPredicate) { - return SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, logger, versionPredicate); - } - public static List indexNames() { return Collections.unmodifiableList(Arrays.asList(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)); } - - /** - * Is the move from {@code previousHealth} to {@code currentHealth} a move from an unhealthy ("RED") index state to a healthy - * ("non-RED") state. - */ - public static boolean isMoveFromRedToNonRed(ClusterIndexHealth previousHealth, ClusterIndexHealth currentHealth) { - return (previousHealth == null || previousHealth.getStatus() == ClusterHealthStatus.RED) - && currentHealth != null && currentHealth.getStatus() != ClusterHealthStatus.RED; - } - - /** - * Is the move from {@code previousHealth} to {@code currentHealth} a move from index-exists to index-deleted - */ - public static boolean isIndexDeleted(ClusterIndexHealth previousHealth, ClusterIndexHealth currentHealth) { - return previousHealth != null && currentHealth == null; - } - } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/BytesKey.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/BytesKey.java index 1534b78899f8b..0ead753a4461c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/BytesKey.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/BytesKey.java @@ -14,7 +14,7 @@ * Simple wrapper around bytes so that it can be used as a cache key. The hashCode is computed * once upon creation and cached. */ -public class BytesKey { +public final class BytesKey { final byte[] bytes; private final int hashCode; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java index 6e97071cea994..b50264a73e949 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java @@ -96,7 +96,7 @@ public static Map getFactories(ThreadPool threadPool, Res map.put(FileRealmSettings.TYPE, config -> new FileRealm(config, resourceWatcherService)); map.put(NativeRealmSettings.TYPE, config -> { final NativeRealm nativeRealm = new NativeRealm(config, nativeUsersStore); - securityLifecycleService.securityIndex().addIndexHealthChangeListener(nativeRealm::onSecurityIndexHealthChange); + securityLifecycleService.securityIndex().addIndexStateListener(nativeRealm::onSecurityIndexStateChange); return nativeRealm; }); map.put(LdapRealmSettings.AD_TYPE, config -> new LdapRealm(LdapRealmSettings.AD_TYPE, config, sslService, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java index 6b8f9eb703db0..c9ccdbb75c0bb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java @@ -6,14 +6,16 @@ package org.elasticsearch.xpack.security.authc.esnative; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; + +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; /** * User/password realm that is backed by an Elasticsearch index @@ -37,12 +39,8 @@ protected void doAuthenticate(UsernamePasswordToken token, ActionListener user) { - char[] hash = users.get(username); + final char[] hash = users.get(username); if (hash == null) { return AuthenticationResult.notHandled(); } @@ -91,7 +91,7 @@ public AuthenticationResult verifyPassword(String username, SecureString passwor } public boolean userExists(String username) { - return users != null && users.containsKey(username); + return users.containsKey(username); } public static Path resolveFile(Environment env) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStore.java index 1631fef60ea89..e17d8c5c7ecfa 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStore.java @@ -75,11 +75,8 @@ int entriesCount() { } public String[] roles(String username) { - if (userRoles == null) { - return Strings.EMPTY_ARRAY; - } - String[] roles = userRoles.get(username); - return roles == null ? Strings.EMPTY_ARRAY : userRoles.get(username); + final String[] roles = userRoles.get(username); + return roles == null ? Strings.EMPTY_ARRAY : roles; } public static Path resolveFile(Environment env) { @@ -160,11 +157,7 @@ public static Map parseFile(Path path, @Nullable Logger logger } for (String user : roleUsers) { - List roles = userToRoles.get(user); - if (roles == null) { - roles = new ArrayList<>(); - userToRoles.put(user, roles); - } + List roles = userToRoles.computeIfAbsent(user, k -> new ArrayList<>()); roles.add(role); } } @@ -185,11 +178,7 @@ public static void writeFile(Map userToRoles, Path path) { HashMap> roleToUsers = new HashMap<>(); for (Map.Entry entry : userToRoles.entrySet()) { for (String role : entry.getValue()) { - List users = roleToUsers.get(role); - if (users == null) { - users = new ArrayList<>(); - roleToUsers.put(role, users); - } + List users = roleToUsers.computeIfAbsent(role, k -> new ArrayList<>()); users.add(entry.getKey()); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java index 93bbe2c1a7567..f8826bebcac71 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticator.java @@ -87,6 +87,14 @@ private SamlAttributes authenticateResponse(Element element, Collection if (logger.isTraceEnabled()) { logger.trace(SamlUtils.describeSamlObject(response)); } + final boolean requireSignedAssertions; + if (response.isSigned()) { + validateSignature(response.getSignature()); + requireSignedAssertions = false; + } else { + requireSignedAssertions = true; + } + if (Strings.hasText(response.getInResponseTo()) && allowedSamlRequestIds.contains(response.getInResponseTo()) == false) { logger.debug("The SAML Response with ID {} is unsolicited. A user might have used a stale URL or the Identity Provider " + "incorrectly populates the InResponseTo attribute", response.getID()); @@ -102,10 +110,10 @@ private SamlAttributes authenticateResponse(Element element, Collection throw samlException("SAML Response is not a 'success' response: Code={} Message={} Detail={}", status.getStatusCode().getValue(), getMessage(status), getDetail(status)); } - + checkIssuer(response.getIssuer(), response); checkResponseDestination(response); - Tuple> details = extractDetails(response, allowedSamlRequestIds); + Tuple> details = extractDetails(response, allowedSamlRequestIds, requireSignedAssertions); final Assertion assertion = details.v1(); final SamlNameId nameId = SamlNameId.forSubject(assertion.getSubject()); final String session = getSessionIndex(assertion); @@ -156,17 +164,8 @@ private void checkResponseDestination(Response response) { } } - private Tuple> extractDetails(Response response, Collection allowedSamlRequestIds) { - final boolean requireSignedAssertions; - if (response.isSigned()) { - validateSignature(response.getSignature()); - requireSignedAssertions = false; - } else { - requireSignedAssertions = true; - } - - checkIssuer(response.getIssuer(), response); - + private Tuple> extractDetails(Response response, Collection allowedSamlRequestIds, + boolean requireSignedAssertions) { final int assertionCount = response.getAssertions().size() + response.getEncryptedAssertions().size(); if (assertionCount > 1) { throw samlException("Expecting only 1 assertion, but response contains multiple (" + assertionCount + ")"); @@ -328,5 +327,4 @@ private void checkLifetimeRestrictions(Conditions conditions) { private void checkLifetimeRestrictions(SubjectConfirmationData subjectConfirmationData) { validateNotOnOrAfter(subjectConfirmationData.getNotOnOrAfter()); } - } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index bd26d778c0d6f..b6df03471242e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.bytes.BytesReference; @@ -38,6 +39,7 @@ import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.io.IOException; import java.io.InputStream; @@ -61,8 +63,8 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.isIndexDeleted; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.isMoveFromRedToNonRed; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; /** * This store reads + writes {@link ExpressionRoleMapping role mappings} in an Elasticsearch @@ -316,17 +318,13 @@ private void reportStats(ActionListener> listener, List void refreshRealms(ActionListener listener, Result result) { String[] realmNames = this.realmsToRefresh.toArray(new String[realmsToRefresh.size()]); final SecurityClient securityClient = new SecurityClient(client); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java index 1f604406a7120..d90f50ca5faa5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -34,6 +35,7 @@ import org.elasticsearch.xpack.core.security.authz.privilege.Privilege; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.security.SecurityLifecycleService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.ArrayList; import java.util.Arrays; @@ -53,8 +55,8 @@ import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.security.SecurityField.setting; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.isIndexDeleted; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.isMoveFromRedToNonRed; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; /** * A composite roles store that combines built in roles, file-based roles, and index-based roles. Checks the built in roles first, then the @@ -323,17 +325,13 @@ public void usageStats(ActionListener> listener) { }, listener::onFailure)); } - public void onSecurityIndexHealthChange(ClusterIndexHealth previousHealth, ClusterIndexHealth currentHealth) { - if (isMoveFromRedToNonRed(previousHealth, currentHealth) || isIndexDeleted(previousHealth, currentHealth)) { + public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { + if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState) || + previousState.isIndexUpToDate != currentState.isIndexUpToDate) { invalidateAll(); } } - public void onSecurityIndexOutOfDateChange(boolean prevOutOfDate, boolean outOfDate) { - assert prevOutOfDate != outOfDate : "this method should only be called if the two values are different"; - invalidateAll(); - } - /** * A mutable class that can be used to represent the combination of one or more {@link IndicesPrivileges} */ diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index bfa02ed17c2c2..4bcfb779b0d50 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -23,6 +23,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -41,6 +42,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.BiConsumer; @@ -53,7 +55,6 @@ import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; -import static org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME; /** * Manages the lifecycle of a single index, its template, mapping and and data upgrades/migrations. @@ -63,16 +64,15 @@ public class SecurityIndexManager extends AbstractComponent { public static final String INTERNAL_SECURITY_INDEX = ".security-" + IndexUpgradeCheckVersion.UPRADE_VERSION; public static final int INTERNAL_INDEX_FORMAT = 6; public static final String SECURITY_VERSION_STRING = "security-version"; - public static final String TEMPLATE_VERSION_PATTERN = - Pattern.quote("${security.template.version}"); + public static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); + public static final String SECURITY_TEMPLATE_NAME = "security-index-template"; private final String indexName; private final Client client; - private final List> indexHealthChangeListeners = new CopyOnWriteArrayList<>(); - private final List> indexOutOfDateListeners = new CopyOnWriteArrayList<>(); + private final List> stateChangeListeners = new CopyOnWriteArrayList<>(); - private volatile State indexState = new State(false, false, false, false, null); + private volatile State indexState = new State(false, false, false, false, null, null); public SecurityIndexManager(Settings settings, Client client, String indexName) { super(settings); @@ -107,81 +107,31 @@ public boolean isMappingUpToDate() { } /** - * Adds a listener which will be notified when the security index health changes. The previous and - * current health will be provided to the listener so that the listener can determine if any action - * needs to be taken. + * Add a listener for notifications on state changes to the configured index. + * + * The previous and current state are provided. */ - public void addIndexHealthChangeListener(BiConsumer listener) { - indexHealthChangeListeners.add(listener); - } - - /** - * Adds a listener which will be notified when the security index out of date value changes. The previous and - * current value will be provided to the listener so that the listener can determine if any action - * needs to be taken. - */ - public void addIndexOutOfDateListener(BiConsumer listener) { - indexOutOfDateListeners.add(listener); + public void addIndexStateListener(BiConsumer listener) { + stateChangeListeners.add(listener); } public void clusterChanged(ClusterChangedEvent event) { - final boolean previousUpToDate = this.indexState.isIndexUpToDate; - processClusterState(event.state()); - checkIndexHealthChange(event); - if (previousUpToDate != this.indexState.isIndexUpToDate) { - notifyIndexOutOfDateListeners(previousUpToDate, this.indexState.isIndexUpToDate); - } - } - - private void processClusterState(ClusterState clusterState) { - assert clusterState != null; - final IndexMetaData securityIndex = resolveConcreteIndex(indexName, clusterState.metaData()); - final boolean indexExists = securityIndex != null; + final State previousState = indexState; + final IndexMetaData indexMetaData = resolveConcreteIndex(indexName, event.state().metaData()); + final boolean indexExists = indexMetaData != null; final boolean isIndexUpToDate = indexExists == false || - INDEX_FORMAT_SETTING.get(securityIndex.getSettings()).intValue() == INTERNAL_INDEX_FORMAT; - final boolean indexAvailable = checkIndexAvailable(clusterState); - final boolean mappingIsUpToDate = indexExists == false || checkIndexMappingUpToDate(clusterState); - final Version mappingVersion = oldestIndexMappingVersion(clusterState); - this.indexState = new State(indexExists, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion); - } - - private void checkIndexHealthChange(ClusterChangedEvent event) { - final ClusterState state = event.state(); - final ClusterState previousState = event.previousState(); - final IndexMetaData indexMetaData = resolveConcreteIndex(indexName, state.metaData()); - final IndexMetaData previousIndexMetaData = resolveConcreteIndex(indexName, previousState.metaData()); - if (indexMetaData != null) { - final ClusterIndexHealth currentHealth = - new ClusterIndexHealth(indexMetaData, state.getRoutingTable().index(indexMetaData.getIndex())); - final ClusterIndexHealth previousHealth = previousIndexMetaData != null ? new ClusterIndexHealth(previousIndexMetaData, - previousState.getRoutingTable().index(previousIndexMetaData.getIndex())) : null; - - if (previousHealth == null || previousHealth.getStatus() != currentHealth.getStatus()) { - notifyIndexHealthChangeListeners(previousHealth, currentHealth); - } - } else if (previousIndexMetaData != null) { - final ClusterIndexHealth previousHealth = - new ClusterIndexHealth(previousIndexMetaData, previousState.getRoutingTable().index(previousIndexMetaData.getIndex())); - notifyIndexHealthChangeListeners(previousHealth, null); - } - } - - private void notifyIndexHealthChangeListeners(ClusterIndexHealth previousHealth, ClusterIndexHealth currentHealth) { - for (BiConsumer consumer : indexHealthChangeListeners) { - try { - consumer.accept(previousHealth, currentHealth); - } catch (Exception e) { - logger.warn(new ParameterizedMessage("failed to notify listener [{}] of index health change", consumer), e); - } - } - } - - private void notifyIndexOutOfDateListeners(boolean previous, boolean current) { - for (BiConsumer consumer : indexOutOfDateListeners) { - try { - consumer.accept(previous, current); - } catch (Exception e) { - logger.warn(new ParameterizedMessage("failed to notify listener [{}] of index out of date change", consumer), e); + INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()).intValue() == INTERNAL_INDEX_FORMAT; + final boolean indexAvailable = checkIndexAvailable(event.state()); + final boolean mappingIsUpToDate = indexExists == false || checkIndexMappingUpToDate(event.state()); + final Version mappingVersion = oldestIndexMappingVersion(event.state()); + final ClusterHealthStatus indexStatus = indexMetaData == null ? null : + new ClusterIndexHealth(indexMetaData, event.state().getRoutingTable().index(indexMetaData.getIndex())).getStatus(); + final State newState = new State(indexExists, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, indexStatus); + this.indexState = newState; + + if (newState.equals(previousState) == false) { + for (BiConsumer listener : stateChangeListeners) { + listener.accept(previousState, newState); } } } @@ -195,7 +145,6 @@ private boolean checkIndexAvailable(ClusterState state) { return false; } - /** * Returns the routing-table for this index, or null if the index does not exist. */ @@ -351,23 +300,59 @@ private Tuple loadMappingAndSettingsSourceFromTemplate() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(SECURITY_TEMPLATE_NAME).source(template, XContentType.JSON); return new Tuple<>(request.mappings().get("doc"), request.settings()); } + + /** + * Return true if the state moves from an unhealthy ("RED") index state to a healthy ("non-RED") state. + */ + public static boolean isMoveFromRedToNonRed(State previousState, State currentState) { + return (previousState.indexStatus == null || previousState.indexStatus == ClusterHealthStatus.RED) + && currentState.indexStatus != null && currentState.indexStatus != ClusterHealthStatus.RED; + } + + /** + * Return true if the state moves from the index existing to the index not existing. + */ + public static boolean isIndexDeleted(State previousState, State currentState) { + return previousState.indexStatus != null && currentState.indexStatus == null; + } + /** - * Holder class so we can update all values at once + * State of the security index. */ - private static class State { - private final boolean indexExists; - private final boolean isIndexUpToDate; - private final boolean indexAvailable; - private final boolean mappingUpToDate; - private final Version mappingVersion; - - private State(boolean indexExists, boolean isIndexUpToDate, boolean indexAvailable, - boolean mappingUpToDate, Version mappingVersion) { + public static class State { + public final boolean indexExists; + public final boolean isIndexUpToDate; + public final boolean indexAvailable; + public final boolean mappingUpToDate; + public final Version mappingVersion; + public final ClusterHealthStatus indexStatus; + + public State(boolean indexExists, boolean isIndexUpToDate, boolean indexAvailable, + boolean mappingUpToDate, Version mappingVersion, ClusterHealthStatus indexStatus) { this.indexExists = indexExists; this.isIndexUpToDate = isIndexUpToDate; this.indexAvailable = indexAvailable; this.mappingUpToDate = mappingUpToDate; this.mappingVersion = mappingVersion; + this.indexStatus = indexStatus; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + State state = (State) o; + return indexExists == state.indexExists && + isIndexUpToDate == state.isIndexUpToDate && + indexAvailable == state.indexAvailable && + mappingUpToDate == state.mappingUpToDate && + Objects.equals(mappingVersion, state.mappingVersion) && + indexStatus == state.indexStatus; + } + + @Override + public int hashCode() { + return Objects.hash(indexExists, isIndexUpToDate, indexAvailable, mappingUpToDate, mappingVersion, indexStatus); } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index 5f082eae62fa9..725508327796a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -12,17 +12,15 @@ import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.security.user.BeatsSystemUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.Before; @@ -63,7 +61,7 @@ protected boolean addMockHttpTransport() { @Override public Set excludeTemplates() { Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME); // don't remove the security index template + templates.add(SecurityIndexManager.SECURITY_TEMPLATE_NAME); // don't remove the security index template return templates; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 7859b95fece48..bc19df6185d63 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -7,6 +7,7 @@ import io.netty.util.ThreadDeathWatcher; import io.netty.util.concurrent.GlobalEventExecutor; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -42,6 +43,7 @@ import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.security.LocalStateSecurity; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -63,7 +65,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.securityIndexMappingSufficientToRead; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsCollectionContaining.hasItem; @@ -470,7 +471,8 @@ public void assertSecurityIndexActive(TestCluster testCluster) throws Exception XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint().startObject(); assertTrue("security index mapping not sufficient to read:\n" + Strings.toString(clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject()), - securityIndexMappingSufficientToRead(clusterState, logger)); + SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, logger, + Version.CURRENT.minimumIndexCompatibilityVersion()::onOrBefore)); Index securityIndex = resolveSecurityIndex(clusterState.metaData()); if (securityIndex != null) { IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(securityIndex); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java deleted file mode 100644 index 02d99fe0edc92..0000000000000 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; - -import org.elasticsearch.Version; -import org.elasticsearch.action.Action; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestBuilder; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.FilterClient; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.TestThreadPool; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.MockTransportClient; -import org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField; -import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; -import org.elasticsearch.xpack.security.test.SecurityTestUtils; -import org.elasticsearch.xpack.core.template.TemplateUtils; -import org.junit.After; -import org.junit.Before; - -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; -import static org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.securityIndexMappingUpToDate; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class SecurityLifecycleServiceTests extends ESTestCase { - private TransportClient transportClient; - private ThreadPool threadPool; - private SecurityLifecycleService securityLifecycleService; - private static final ClusterState EMPTY_CLUSTER_STATE = - new ClusterState.Builder(new ClusterName("test-cluster")).build(); - private CopyOnWriteArrayList listeners; - - @Before - public void setup() { - DiscoveryNode localNode = mock(DiscoveryNode.class); - when(localNode.getHostAddress()).thenReturn(buildNewFakeTransportAddress().toString()); - ClusterService clusterService = mock(ClusterService.class); - when(clusterService.localNode()).thenReturn(localNode); - - threadPool = new TestThreadPool("security template service tests"); - transportClient = new MockTransportClient(Settings.EMPTY); - Client client = new FilterClient(transportClient) { - @Override - protected > - void doExecute(Action action, Request request, - ActionListener listener) { - listeners.add(listener); - } - }; - securityLifecycleService = new SecurityLifecycleService(Settings.EMPTY, clusterService, - threadPool, client, mock(IndexAuditTrail.class)); - listeners = new CopyOnWriteArrayList<>(); - } - - @After - public void stop() throws InterruptedException { - if (transportClient != null) { - transportClient.close(); - } - terminate(threadPool); - } - - public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { - ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate( - "/" + SECURITY_TEMPLATE_NAME + ".json" - ); - securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", - clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - // No upgrade actions run - assertThat(listeners.size(), equalTo(0)); - } - - public void testIndexTemplateVersionMatching() throws Exception { - String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); - final ClusterState clusterState = clusterStateBuilder.build(); - - assertTrue(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME, clusterState, logger, - Version.V_5_0_0::before)); - assertFalse(SecurityIndexManager.checkTemplateExistsAndVersionMatches( - SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME, clusterState, logger, - Version.V_5_0_0::after)); - } - - public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { - String securityTemplateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(securityTemplateString); - securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", - clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - assertThat(listeners.size(), equalTo(0)); - } - - public void testMappingVersionMatching() throws IOException { - String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); - securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", - clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); - final SecurityIndexManager securityIndex = securityLifecycleService.securityIndex(); - assertTrue(securityIndex.checkMappingVersion(Version.V_5_0_0::before)); - assertFalse(securityIndex.checkMappingVersion(Version.V_5_0_0::after)); - } - - public void testMissingVersionMappingThrowsError() throws IOException { - String templateString = "/missing-version-" + SECURITY_TEMPLATE_NAME + ".json"; - ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); - final ClusterState clusterState = clusterStateBuilder.build(); - IllegalStateException exception = expectThrows(IllegalStateException.class, - () -> securityIndexMappingUpToDate(clusterState, logger)); - assertEquals("Cannot read security-version string in index " + SECURITY_INDEX_NAME, - exception.getMessage()); - } - - public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { - final ClusterName clusterName = new ClusterName("test-cluster"); - final ClusterState.Builder clusterStateBuilder = ClusterState.builder(clusterName); - String mappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; - IndexTemplateMetaData.Builder templateMeta = getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, mappingString); - MetaData.Builder builder = new MetaData.Builder(clusterStateBuilder.build().getMetaData()); - builder.put(templateMeta); - clusterStateBuilder.metaData(builder); - securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() - , EMPTY_CLUSTER_STATE)); - assertThat(listeners.size(), equalTo(0)); - } - - private ClusterState.Builder createClusterStateWithMapping(String securityTemplateString) throws IOException { - final ClusterState clusterState = createClusterStateWithIndex(securityTemplateString).build(); - final String indexName = clusterState.metaData().getAliasAndIndexLookup() - .get(SECURITY_INDEX_NAME).getIndices().get(0).getIndex().getName(); - return ClusterState.builder(clusterState).routingTable(SecurityTestUtils.buildIndexRoutingTable(indexName)); - } - - private ClusterState.Builder createClusterStateWithMappingAndTemplate(String securityTemplateString) throws IOException { - ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(securityTemplateString); - MetaData.Builder metaDataBuilder = new MetaData.Builder(clusterStateBuilder.build().metaData()); - String securityMappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; - IndexTemplateMetaData.Builder securityTemplateMeta = getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, securityMappingString); - metaDataBuilder.put(securityTemplateMeta); - return clusterStateBuilder.metaData(metaDataBuilder); - } - - private static IndexMetaData.Builder createIndexMetadata(String indexName, String templateString) throws IOException { - String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_PATTERN); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(template, XContentType.JSON); - IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName); - indexMetaData.settings(Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .build()); - - for (Map.Entry entry : request.mappings().entrySet()) { - indexMetaData.putMapping(entry.getKey(), entry.getValue()); - } - return indexMetaData; - } - - public ClusterState.Builder createClusterStateWithTemplate(String securityTemplateString) throws IOException { - // add the correct mapping no matter what the template - ClusterState clusterState = createClusterStateWithIndex("/" + SECURITY_TEMPLATE_NAME + ".json").build(); - final MetaData.Builder metaDataBuilder = new MetaData.Builder(clusterState.metaData()); - metaDataBuilder.put(getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, securityTemplateString)); - return ClusterState.builder(clusterState).metaData(metaDataBuilder); - } - - private ClusterState.Builder createClusterStateWithIndex(String securityTemplate) throws IOException { - final MetaData.Builder metaDataBuilder = new MetaData.Builder(); - final boolean withAlias = randomBoolean(); - final String securityIndexName = SECURITY_INDEX_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); - metaDataBuilder.put(createIndexMetadata(securityIndexName, securityTemplate)); - - ClusterState.Builder clusterStateBuilder = ClusterState.builder(state()); - if (withAlias) { - // try with .security index as an alias - clusterStateBuilder.metaData(SecurityTestUtils.addAliasToMetaData(metaDataBuilder.build(), securityIndexName)); - } else { - // try with .security index as a concrete index - clusterStateBuilder.metaData(metaDataBuilder); - } - - clusterStateBuilder.routingTable(SecurityTestUtils.buildIndexRoutingTable(securityIndexName)); - return clusterStateBuilder; - } - - private static IndexTemplateMetaData.Builder getIndexTemplateMetaData( - String templateName, String templateString) throws IOException { - - String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_PATTERN); - PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(template, XContentType.JSON); - IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(templateName) - .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))); - for (Map.Entry entry : request.mappings().entrySet()) { - templateBuilder.putMapping(entry.getKey(), entry.getValue()); - } - return templateBuilder; - } - - // cluster state where local node is master - private static ClusterState state() { - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); - discoBuilder.masterNodeId("1"); - discoBuilder.localNodeId("1"); - ClusterState.Builder state = ClusterState.builder(new ClusterName("test-cluster")); - state.nodes(discoBuilder); - state.metaData(MetaData.builder().generateClusterUuidIfNeeded()); - return state.build(); - } -} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java index a1e8cc3c4e993..ec448f14e9160 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; @@ -52,6 +51,7 @@ import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.LocalStateSecurity; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.Message; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import org.joda.time.DateTime; @@ -254,7 +254,7 @@ public void afterTest() { @Override protected Set excludeTemplates() { - return Sets.newHashSet(SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME, IndexAuditTrail.INDEX_TEMPLATE_NAME); + return Sets.newHashSet(SecurityIndexManager.SECURITY_TEMPLATE_NAME, IndexAuditTrail.INDEX_TEMPLATE_NAME); } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java index 5b90b2e1e4609..7002803a3d49c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/RemoteIndexAuditTrailStartingTests.java @@ -18,9 +18,9 @@ import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.audit.AuditTrailService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.Before; @@ -70,7 +70,7 @@ public Settings nodeSettings(int nodeOrdinal) { @Override protected Set excludeTemplates() { - return Sets.newHashSet(SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME, IndexAuditTrail.INDEX_TEMPLATE_NAME); + return Sets.newHashSet(SecurityIndexManager.SECURITY_TEMPLATE_NAME, IndexAuditTrail.INDEX_TEMPLATE_NAME); } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java index f0af7a2539e42..47eb1eabae159 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java @@ -47,10 +47,10 @@ public void testNativeRealmRegistersIndexHealthChangeListener() throws Exception Settings settings = Settings.builder().put("path.home", createTempDir()).build(); factories.get(NativeRealmSettings.TYPE).create(new RealmConfig("test", Settings.EMPTY, settings, TestEnvironment.newEnvironment(settings), new ThreadContext(settings))); - verify(securityIndex).addIndexHealthChangeListener(isA(BiConsumer.class)); + verify(securityIndex).addIndexStateListener(isA(BiConsumer.class)); factories.get(NativeRealmSettings.TYPE).create(new RealmConfig("test", Settings.EMPTY, settings, TestEnvironment.newEnvironment(settings), new ThreadContext(settings))); - verify(securityIndex, times(2)).addIndexHealthChangeListener(isA(BiConsumer.class)); + verify(securityIndex, times(2)).addIndexStateListener(isA(BiConsumer.class)); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index 8b64ad4b1ec56..7e2d5242101c1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.RealmConfig; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.concurrent.atomic.AtomicInteger; @@ -20,6 +21,10 @@ public class NativeRealmTests extends ESTestCase { + private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { + return new SecurityIndexManager.State(true, true, true, true, null, indexStatus); + } + public void testCacheClearOnIndexHealthChange() { final AtomicInteger numInvalidation = new AtomicInteger(0); int expectedInvalidation = 0; @@ -34,34 +39,34 @@ void clearCache() { }; // existing to no longer present - ClusterIndexHealth previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - ClusterIndexHealth currentHealth = null; - nativeRealm.onSecurityIndexHealthChange(previousHealth, currentHealth); + SecurityIndexManager.State previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + SecurityIndexManager.State currentState = dummyState(null); + nativeRealm.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // doesn't exist to exists - previousHealth = null; - currentHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - nativeRealm.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(null); + currentState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + nativeRealm.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // green or yellow to red - previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - currentHealth = getClusterIndexHealth(ClusterHealthStatus.RED); - nativeRealm.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + currentState = dummyState(ClusterHealthStatus.RED); + nativeRealm.onSecurityIndexStateChange(previousState, currentState); assertEquals(expectedInvalidation, numInvalidation.get()); // red to non red - previousHealth = getClusterIndexHealth(ClusterHealthStatus.RED); - currentHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - nativeRealm.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(ClusterHealthStatus.RED); + currentState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + nativeRealm.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // green to yellow or yellow to green - previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - currentHealth = getClusterIndexHealth( - previousHealth.getStatus() == ClusterHealthStatus.GREEN ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN); - nativeRealm.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + currentState = dummyState(previousState.indexStatus == ClusterHealthStatus.GREEN ? + ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN); + nativeRealm.onSecurityIndexStateChange(previousState, currentState); assertEquals(expectedInvalidation, numInvalidation.get()); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 3a67ab9447e32..693118c21bde5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.env.Environment; @@ -41,7 +40,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.security.test.SecurityTestUtils.getClusterIndexHealth; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; @@ -128,6 +126,9 @@ private String randomiseDn(String dn) { return dn; } + private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { + return new SecurityIndexManager.State(true, true, true, true, null, indexStatus); + } public void testCacheClearOnIndexHealthChange() { final AtomicInteger numInvalidation = new AtomicInteger(0); @@ -135,34 +136,34 @@ public void testCacheClearOnIndexHealthChange() { int expectedInvalidation = 0; // existing to no longer present - ClusterIndexHealth previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - ClusterIndexHealth currentHealth = null; - store.onSecurityIndexHealthChange(previousHealth, currentHealth); + SecurityIndexManager.State previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + SecurityIndexManager.State currentState = dummyState(null); + store.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // doesn't exist to exists - previousHealth = null; - currentHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - store.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(null); + currentState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + store.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // green or yellow to red - previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - currentHealth = getClusterIndexHealth(ClusterHealthStatus.RED); - store.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + currentState = dummyState(ClusterHealthStatus.RED); + store.onSecurityIndexStateChange(previousState, currentState); assertEquals(expectedInvalidation, numInvalidation.get()); // red to non red - previousHealth = getClusterIndexHealth(ClusterHealthStatus.RED); - currentHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - store.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(ClusterHealthStatus.RED); + currentState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + store.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // green to yellow or yellow to green - previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - currentHealth = getClusterIndexHealth( - previousHealth.getStatus() == ClusterHealthStatus.GREEN ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN); - store.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + currentState = dummyState(previousState.indexStatus == ClusterHealthStatus.GREEN ? + ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN); + store.onSecurityIndexStateChange(previousState, currentState); assertEquals(expectedInvalidation, numInvalidation.get()); } @@ -170,10 +171,14 @@ public void testCacheClearOnIndexOutOfDateChange() { final AtomicInteger numInvalidation = new AtomicInteger(0); final NativeRoleMappingStore store = buildRoleMappingStoreForInvalidationTesting(numInvalidation); - store.onSecurityIndexOutOfDateChange(false, true); + store.onSecurityIndexStateChange( + new SecurityIndexManager.State(true, false, true, true, null, null), + new SecurityIndexManager.State(true, true, true, true, null, null)); assertEquals(1, numInvalidation.get()); - store.onSecurityIndexOutOfDateChange(true, false); + store.onSecurityIndexStateChange( + new SecurityIndexManager.State(true, true, true, true, null, null), + new SecurityIndexManager.State(true, false, true, true, null, null)); assertEquals(2, numInvalidation.get()); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 7c732cd7c52f9..ff9d93b3ba818 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.io.IOException; import java.util.Arrays; @@ -451,6 +452,10 @@ public void testCustomRolesProvidersLicensing() { assertEquals(0, role.indices().groups().length); } + private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { + return new SecurityIndexManager.State(true, true, true, true, null, indexStatus); + } + public void testCacheClearOnIndexHealthChange() { final AtomicInteger numInvalidation = new AtomicInteger(0); @@ -465,34 +470,34 @@ public void invalidateAll() { int expectedInvalidation = 0; // existing to no longer present - ClusterIndexHealth previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - ClusterIndexHealth currentHealth = null; - compositeRolesStore.onSecurityIndexHealthChange(previousHealth, currentHealth); + SecurityIndexManager.State previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + SecurityIndexManager.State currentState = dummyState(null); + compositeRolesStore.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // doesn't exist to exists - previousHealth = null; - currentHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - compositeRolesStore.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(null); + currentState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + compositeRolesStore.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // green or yellow to red - previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - currentHealth = getClusterIndexHealth(ClusterHealthStatus.RED); - compositeRolesStore.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + currentState = dummyState(ClusterHealthStatus.RED); + compositeRolesStore.onSecurityIndexStateChange(previousState, currentState); assertEquals(expectedInvalidation, numInvalidation.get()); // red to non red - previousHealth = getClusterIndexHealth(ClusterHealthStatus.RED); - currentHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - compositeRolesStore.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(ClusterHealthStatus.RED); + currentState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + compositeRolesStore.onSecurityIndexStateChange(previousState, currentState); assertEquals(++expectedInvalidation, numInvalidation.get()); // green to yellow or yellow to green - previousHealth = getClusterIndexHealth(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); - currentHealth = getClusterIndexHealth( - previousHealth.getStatus() == ClusterHealthStatus.GREEN ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN); - compositeRolesStore.onSecurityIndexHealthChange(previousHealth, currentHealth); + previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW)); + currentState = dummyState(previousState.indexStatus == ClusterHealthStatus.GREEN ? + ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN); + compositeRolesStore.onSecurityIndexStateChange(previousState, currentState); assertEquals(expectedInvalidation, numInvalidation.get()); } @@ -508,10 +513,14 @@ public void invalidateAll() { } }; - compositeRolesStore.onSecurityIndexOutOfDateChange(false, true); + compositeRolesStore.onSecurityIndexStateChange( + new SecurityIndexManager.State(true, false, true, true, null, null), + new SecurityIndexManager.State(true, true, true, true, null, null)); assertEquals(1, numInvalidation.get()); - compositeRolesStore.onSecurityIndexOutOfDateChange(true, false); + compositeRolesStore.onSecurityIndexStateChange( + new SecurityIndexManager.State(true, true, true, true, null, null), + new SecurityIndexManager.State(true, false, true, true, null, null)); assertEquals(2, numInvalidation.get()); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index 8df2b14e3ba8e..ab6664b53b0fb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -38,12 +38,12 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField; import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.junit.After; import org.junit.Before; @@ -260,7 +260,7 @@ private ClusterState getClusterStateWithSecurityIndex() { .build(); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder(securityIndexName).settings(settings)) - .put(new IndexTemplateMetaData(SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME, 0, 0, + .put(new IndexTemplateMetaData(SecurityIndexManager.SECURITY_TEMPLATE_NAME, 0, 0, Collections.singletonList(securityIndexName), Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of(), ImmutableOpenMap.of())) .build(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index e85c8629f2c80..fe51f2beca34d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -52,15 +51,18 @@ import org.junit.Before; import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; +import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN; +import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class SecurityIndexManagerTests extends ESTestCase { - private static final ClusterName CLUSTER_NAME = new ClusterName("index-lifecycle-manager-tests"); + private static final ClusterName CLUSTER_NAME = new ClusterName("security-index-manager-tests"); private static final ClusterState EMPTY_CLUSTER_STATE = new ClusterState.Builder(CLUSTER_NAME).build(); - public static final String INDEX_NAME = "SecurityIndexManagerTests"; + public static final String INDEX_NAME = ".security"; private static final String TEMPLATE_NAME = "SecurityIndexManagerTests-template"; private SecurityIndexManager manager; private Map, Map>> actions; @@ -127,29 +129,14 @@ private ClusterChangedEvent event(ClusterState.Builder clusterStateBuilder) { public void testIndexHealthChangeListeners() throws Exception { final AtomicBoolean listenerCalled = new AtomicBoolean(false); - final AtomicReference previousHealth = new AtomicReference<>(); - final AtomicReference currentHealth = new AtomicReference<>(); - final BiConsumer listener = (prevState, state) -> { - previousHealth.set(prevState); - currentHealth.set(state); + final AtomicReference previousState = new AtomicReference<>(); + final AtomicReference currentState = new AtomicReference<>(); + final BiConsumer listener = (prevState, state) -> { + previousState.set(prevState); + currentState.set(state); listenerCalled.set(true); }; - - if (randomBoolean()) { - if (randomBoolean()) { - manager.addIndexHealthChangeListener(listener); - manager.addIndexHealthChangeListener((prevState, state) -> { - throw new RuntimeException("throw after listener"); - }); - } else { - manager.addIndexHealthChangeListener((prevState, state) -> { - throw new RuntimeException("throw before listener"); - }); - manager.addIndexHealthChangeListener(listener); - } - } else { - manager.addIndexHealthChangeListener(listener); - } + manager.addIndexStateListener(listener); // index doesn't exist and now exists final ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME); @@ -157,26 +144,26 @@ public void testIndexHealthChangeListeners() throws Exception { manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); - assertNull(previousHealth.get()); - assertEquals(ClusterHealthStatus.GREEN, currentHealth.get().getStatus()); + assertNull(previousState.get().indexStatus); + assertEquals(ClusterHealthStatus.GREEN, currentState.get().indexStatus); // reset and call with no change to the index listenerCalled.set(false); - previousHealth.set(null); - currentHealth.set(null); + previousState.set(null); + currentState.set(null); ClusterChangedEvent event = new ClusterChangedEvent("same index health", clusterStateBuilder.build(), clusterStateBuilder.build()); manager.clusterChanged(event); assertFalse(listenerCalled.get()); - assertNull(previousHealth.get()); - assertNull(currentHealth.get()); + assertNull(previousState.get()); + assertNull(currentState.get()); // index with different health listenerCalled.set(false); - previousHealth.set(null); - currentHealth.set(null); - ClusterState previousState = clusterStateBuilder.build(); - Index prevIndex = previousState.getRoutingTable().index(INDEX_NAME).getIndex(); + previousState.set(null); + currentState.set(null); + ClusterState previousClusterState = clusterStateBuilder.build(); + Index prevIndex = previousClusterState.getRoutingTable().index(INDEX_NAME).getIndex(); clusterStateBuilder.routingTable(RoutingTable.builder() .add(IndexRoutingTable.builder(prevIndex) .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(prevIndex, 0)) @@ -189,29 +176,30 @@ public void testIndexHealthChangeListeners() throws Exception { - event = new ClusterChangedEvent("different index health", clusterStateBuilder.build(), previousState); + event = new ClusterChangedEvent("different index health", clusterStateBuilder.build(), previousClusterState); manager.clusterChanged(event); assertTrue(listenerCalled.get()); - assertEquals(ClusterHealthStatus.GREEN, previousHealth.get().getStatus()); - assertEquals(ClusterHealthStatus.RED, currentHealth.get().getStatus()); + assertEquals(ClusterHealthStatus.GREEN, previousState.get().indexStatus); + assertEquals(ClusterHealthStatus.RED, currentState.get().indexStatus); // swap prev and current listenerCalled.set(false); - previousHealth.set(null); - currentHealth.set(null); - event = new ClusterChangedEvent("different index health swapped", previousState, clusterStateBuilder.build()); + previousState.set(null); + currentState.set(null); + event = new ClusterChangedEvent("different index health swapped", previousClusterState, clusterStateBuilder.build()); manager.clusterChanged(event); assertTrue(listenerCalled.get()); - assertEquals(ClusterHealthStatus.RED, previousHealth.get().getStatus()); - assertEquals(ClusterHealthStatus.GREEN, currentHealth.get().getStatus()); + assertEquals(ClusterHealthStatus.RED, previousState.get().indexStatus); + assertEquals(ClusterHealthStatus.GREEN, currentState.get().indexStatus); } public void testIndexOutOfDateListeners() throws Exception { final AtomicBoolean listenerCalled = new AtomicBoolean(false); manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME))); - manager.addIndexOutOfDateListener((prev, current) -> { + AtomicBoolean upToDateChanged = new AtomicBoolean(); + manager.addIndexStateListener((prev, current) -> { listenerCalled.set(true); - assertNotEquals(prev, current); + upToDateChanged.set(prev.isIndexUpToDate != current.isIndexUpToDate); }); assertTrue(manager.isIndexUpToDate()); @@ -225,12 +213,14 @@ public void testIndexOutOfDateListeners() throws Exception { markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); assertTrue(listenerCalled.get()); + assertTrue(upToDateChanged.get()); assertFalse(manager.isIndexUpToDate()); listenerCalled.set(false); assertFalse(listenerCalled.get()); manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME))); assertTrue(listenerCalled.get()); + assertTrue(upToDateChanged.get()); assertTrue(manager.isIndexUpToDate()); listenerCalled.set(false); @@ -238,7 +228,8 @@ public void testIndexOutOfDateListeners() throws Exception { clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME, SecurityIndexManager.INTERNAL_INDEX_FORMAT); markShardsAvailable(clusterStateBuilder); manager.clusterChanged(event(clusterStateBuilder)); - assertFalse(listenerCalled.get()); + assertTrue(listenerCalled.get()); + assertFalse(upToDateChanged.get()); assertTrue(manager.isIndexUpToDate()); } @@ -324,4 +315,139 @@ private static String loadTemplate(String templateName) { final String resource = "/" + templateName + ".json"; return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN); } + + public void testMappingVersionMatching() throws IOException { + String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); + manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); + assertTrue(manager.checkMappingVersion(Version.CURRENT.minimumIndexCompatibilityVersion()::before)); + assertFalse(manager.checkMappingVersion(Version.CURRENT.minimumIndexCompatibilityVersion()::after)); + } + + public void testMissingVersionMappingThrowsError() throws IOException { + String templateString = "/missing-version-" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(templateString); + final ClusterState clusterState = clusterStateBuilder.build(); + IllegalStateException exception = expectThrows(IllegalStateException.class, + () -> SecurityIndexManager.checkIndexMappingVersionMatches(SECURITY_INDEX_NAME, clusterState, logger, Version.CURRENT::equals)); + assertEquals("Cannot read security-version string in index " + SECURITY_INDEX_NAME, exception.getMessage()); + } + + public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException { + ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate( + "/" + SECURITY_TEMPLATE_NAME + ".json" + ); + manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); + // No upgrade actions run + assertThat(actions.size(), equalTo(0)); + } + + public void testIndexTemplateVersionMatching() throws Exception { + String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString); + final ClusterState clusterState = clusterStateBuilder.build(); + + assertTrue(SecurityIndexManager.checkTemplateExistsAndVersionMatches( + SecurityIndexManager.SECURITY_TEMPLATE_NAME, clusterState, logger, + Version.V_5_0_0::before)); + assertFalse(SecurityIndexManager.checkTemplateExistsAndVersionMatches( + SecurityIndexManager.SECURITY_TEMPLATE_NAME, clusterState, logger, + Version.V_5_0_0::after)); + } + + public void testUpToDateMappingsAreIdentifiedAsUpToDate() throws IOException { + String securityTemplateString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + ClusterState.Builder clusterStateBuilder = createClusterStateWithMappingAndTemplate(securityTemplateString); + manager.clusterChanged(new ClusterChangedEvent("test-event", + clusterStateBuilder.build(), EMPTY_CLUSTER_STATE)); + assertThat(actions.size(), equalTo(0)); + } + + public void testMissingIndexIsIdentifiedAsUpToDate() throws IOException { + final ClusterName clusterName = new ClusterName("test-cluster"); + final ClusterState.Builder clusterStateBuilder = ClusterState.builder(clusterName); + String mappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + IndexTemplateMetaData.Builder templateMeta = getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, mappingString); + MetaData.Builder builder = new MetaData.Builder(clusterStateBuilder.build().getMetaData()); + builder.put(templateMeta); + clusterStateBuilder.metaData(builder); + manager.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build() + , EMPTY_CLUSTER_STATE)); + assertThat(actions.size(), equalTo(0)); + } + + private ClusterState.Builder createClusterStateWithTemplate(String securityTemplateString) throws IOException { + // add the correct mapping no matter what the template + ClusterState clusterState = createClusterStateWithIndex("/" + SECURITY_TEMPLATE_NAME + ".json").build(); + final MetaData.Builder metaDataBuilder = new MetaData.Builder(clusterState.metaData()); + metaDataBuilder.put(getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, securityTemplateString)); + return ClusterState.builder(clusterState).metaData(metaDataBuilder); + } + + private ClusterState.Builder createClusterStateWithMapping(String securityTemplateString) throws IOException { + final ClusterState clusterState = createClusterStateWithIndex(securityTemplateString).build(); + final String indexName = clusterState.metaData().getAliasAndIndexLookup() + .get(SECURITY_INDEX_NAME).getIndices().get(0).getIndex().getName(); + return ClusterState.builder(clusterState).routingTable(SecurityTestUtils.buildIndexRoutingTable(indexName)); + } + + private ClusterState.Builder createClusterStateWithMappingAndTemplate(String securityTemplateString) throws IOException { + ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(securityTemplateString); + MetaData.Builder metaDataBuilder = new MetaData.Builder(clusterStateBuilder.build().metaData()); + String securityMappingString = "/" + SECURITY_TEMPLATE_NAME + ".json"; + IndexTemplateMetaData.Builder securityTemplateMeta = getIndexTemplateMetaData(SECURITY_TEMPLATE_NAME, securityMappingString); + metaDataBuilder.put(securityTemplateMeta); + return clusterStateBuilder.metaData(metaDataBuilder); + } + + private static IndexMetaData.Builder createIndexMetadata(String indexName, String templateString) throws IOException { + String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), + SecurityIndexManager.TEMPLATE_VERSION_PATTERN); + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + request.source(template, XContentType.JSON); + IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName); + indexMetaData.settings(Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .build()); + + for (Map.Entry entry : request.mappings().entrySet()) { + indexMetaData.putMapping(entry.getKey(), entry.getValue()); + } + return indexMetaData; + } + + private ClusterState.Builder createClusterStateWithIndex(String securityTemplate) throws IOException { + final MetaData.Builder metaDataBuilder = new MetaData.Builder(); + final boolean withAlias = randomBoolean(); + final String securityIndexName = SECURITY_INDEX_NAME + (withAlias ? "-" + randomAlphaOfLength(5) : ""); + metaDataBuilder.put(createIndexMetadata(securityIndexName, securityTemplate)); + + ClusterState.Builder clusterStateBuilder = ClusterState.builder(state()); + if (withAlias) { + // try with .security index as an alias + clusterStateBuilder.metaData(SecurityTestUtils.addAliasToMetaData(metaDataBuilder.build(), securityIndexName)); + } else { + // try with .security index as a concrete index + clusterStateBuilder.metaData(metaDataBuilder); + } + + clusterStateBuilder.routingTable(SecurityTestUtils.buildIndexRoutingTable(securityIndexName)); + return clusterStateBuilder; + } + + private static IndexTemplateMetaData.Builder getIndexTemplateMetaData(String templateName, String templateString) throws IOException { + + String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), + SecurityIndexManager.TEMPLATE_VERSION_PATTERN); + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + request.source(template, XContentType.JSON); + IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(templateName) + .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))); + for (Map.Entry entry : request.mappings().entrySet()) { + templateBuilder.putMapping(entry.getKey(), entry.getValue()); + } + return templateBuilder; + } } \ No newline at end of file diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java index c46bac7e6efbc..075ce2772b394 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java @@ -22,11 +22,14 @@ import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.transport.Transport; import org.elasticsearch.xpack.core.TestXPackTransportClient; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.common.socket.SocketAccess; +import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.LocalStateSecurity; @@ -39,7 +42,12 @@ import java.nio.charset.StandardCharsets; import java.security.KeyStore; import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Locale; +import java.util.Set; import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForStore; import static org.hamcrest.CoreMatchers.is; @@ -65,12 +73,18 @@ protected boolean transportSSLEnabled() { } // no SSL exception as this is the exception is returned when connecting - public void testThatUnconfiguredCiphersAreRejected() { + public void testThatUnconfiguredCiphersAreRejected() throws Exception { + Set supportedCiphers = Sets.newHashSet(SSLContext.getDefault().getSupportedSSLParameters().getCipherSuites()); + Set defaultXPackCiphers = Sets.newHashSet(XPackSettings.DEFAULT_CIPHERS); + final List unconfiguredCiphers = new ArrayList<>(Sets.difference(supportedCiphers, defaultXPackCiphers)); + Collections.shuffle(unconfiguredCiphers, random()); + assumeFalse("the unconfigured ciphers list is empty", unconfiguredCiphers.isEmpty()); + try (TransportClient transportClient = new TestXPackTransportClient(Settings.builder() .put(transportClientSettings()) .put("node.name", "programmatic_transport_client") .put("cluster.name", internalCluster().getClusterName()) - .putList("xpack.ssl.cipher_suites", "TLS_ECDH_anon_WITH_RC4_128_SHA", "SSL_RSA_WITH_3DES_EDE_CBC_SHA") + .putList("xpack.ssl.cipher_suites", unconfiguredCiphers) .build(), LocalStateSecurity.class)) { TransportAddress transportAddress = randomFrom(internalCluster().getInstance(Transport.class).boundAddress().boundAddresses()); diff --git a/x-pack/plugin/security/src/test/resources/SecurityIndexManagerTests-template.json b/x-pack/plugin/security/src/test/resources/SecurityIndexManagerTests-template.json index d9a53e4622f5f..0957b1da7ec70 100644 --- a/x-pack/plugin/security/src/test/resources/SecurityIndexManagerTests-template.json +++ b/x-pack/plugin/security/src/test/resources/SecurityIndexManagerTests-template.json @@ -1,5 +1,5 @@ { - "index_patterns": "IndexLifeCycleManagerTests", + "index_patterns": ".security", "mappings": { "doc": { "_meta": { diff --git a/x-pack/plugin/sql/jdbc/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 20da7d0f78e9d..0000000000000 --- a/x-pack/plugin/sql/jdbc/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0b06e4f6514256a3f187a9892e520638b9c59e63 \ No newline at end of file diff --git a/x-pack/plugin/sql/jdbc/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 b/x-pack/plugin/sql/jdbc/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..c25718d0a9ee0 --- /dev/null +++ b/x-pack/plugin/sql/jdbc/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +5167fb0a14434cb10ec3224e9e32ca668e9f9ad4 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-proto/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 deleted file mode 100644 index 20da7d0f78e9d..0000000000000 --- a/x-pack/plugin/sql/sql-proto/licenses/lucene-core-7.4.0-snapshot-1ed95c097b.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0b06e4f6514256a3f187a9892e520638b9c59e63 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-proto/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 new file mode 100644 index 0000000000000..c25718d0a9ee0 --- /dev/null +++ b/x-pack/plugin/sql/sql-proto/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1 @@ -0,0 +1 @@ +5167fb0a14434cb10ec3224e9e32ca668e9f9ad4 \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java index 95c9ade5e295d..c024af48187d3 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java @@ -25,8 +25,8 @@ public enum DataType { SHORT( JDBCType.SMALLINT, Short.class, Short.BYTES, 5, 6, true, false, true), INTEGER( JDBCType.INTEGER, Integer.class, Integer.BYTES, 10, 11, true, false, true), LONG( JDBCType.BIGINT, Long.class, Long.BYTES, 19, 20, true, false, true), - // 53 bits defaultPrecision ~ 16(15.95) decimal digits (53log10(2)), - DOUBLE( JDBCType.DOUBLE, Double.class, Double.BYTES, 16, 25, false, true, true), + // 53 bits defaultPrecision ~ 15(15.95) decimal digits (53log10(2)), + DOUBLE( JDBCType.DOUBLE, Double.class, Double.BYTES, 15, 25, false, true, true), // 24 bits defaultPrecision - 24*log10(2) =~ 7 (7.22) FLOAT( JDBCType.REAL, Float.class, Float.BYTES, 7, 15, false, true, true), HALF_FLOAT( JDBCType.FLOAT, Double.class, Double.BYTES, 16, 25, false, true, true), @@ -37,7 +37,10 @@ public enum DataType { OBJECT( JDBCType.STRUCT, null, -1, 0, 0), NESTED( JDBCType.STRUCT, null, -1, 0, 0), BINARY( JDBCType.VARBINARY, byte[].class, -1, Integer.MAX_VALUE, 0), - DATE( JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES, 19, 20); + // since ODBC and JDBC interpret precision for Date as display size, + // the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone) + // see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288 + DATE( JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES, 24, 24); // @formatter:on private static final Map jdbcToEs; @@ -75,7 +78,7 @@ public enum DataType { *

* Specified column size. For numeric data, this is the maximum precision. For character * data, this is the length in characters. For datetime datatypes, this is the length in characters of the - * String representation (assuming the maximum allowed defaultPrecision of the fractional seconds component). + * String representation (assuming the maximum allowed defaultPrecision of the fractional milliseconds component). */ public final int defaultPrecision; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/CommandBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/CommandBuilder.java index fb08d08fcb926..bf432a7236357 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/CommandBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/CommandBuilder.java @@ -148,6 +148,7 @@ public Object visitSysCatalogs(SysCatalogsContext ctx) { @Override public SysTables visitSysTables(SysTablesContext ctx) { List types = new ArrayList<>(); + boolean legacyTableType = false; for (StringContext string : ctx.string()) { String value = string(string); if (value != null) { @@ -156,6 +157,12 @@ public SysTables visitSysTables(SysTablesContext ctx) { // since % is the same as not specifying a value, choose // https://docs.microsoft.com/en-us/sql/odbc/reference/develop-app/value-list-arguments?view=ssdt-18vs2017 // that is skip the value + } + // special case for legacy apps (like msquery) that always asks for 'TABLE' + // which we manually map to all concrete tables supported + else if (value.toUpperCase(Locale.ROOT).equals("TABLE")) { + legacyTableType = true; + types.add(IndexType.INDEX); } else { IndexType type = IndexType.from(value); types.add(type); @@ -165,7 +172,7 @@ public SysTables visitSysTables(SysTablesContext ctx) { // if the ODBC enumeration is specified, skip validation EnumSet set = types.isEmpty() ? null : EnumSet.copyOf(types); - return new SysTables(source(ctx), visitPattern(ctx.clusterPattern), visitPattern(ctx.tablePattern), set); + return new SysTables(source(ctx), visitPattern(ctx.clusterPattern), visitPattern(ctx.tablePattern), set, legacyTableType); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 3c01736cebe89..8005ce0758981 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.tree.NodeInfo; import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; import org.elasticsearch.xpack.sql.type.EsField; import java.sql.DatabaseMetaData; @@ -29,7 +30,6 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.sql.type.DataType.INTEGER; -import static org.elasticsearch.xpack.sql.type.DataType.NULL; import static org.elasticsearch.xpack.sql.type.DataType.SHORT; /** @@ -133,11 +133,7 @@ static void fillInRows(String clusterName, String indexName, Map output() { @Override public final void execute(SqlSession session, ActionListener listener) { listener.onResponse(Rows.of(output(), IndexType.VALID.stream() + // *DBC requires ascending order + .sorted(Comparator.comparing(t -> t.toSql())) .map(t -> singletonList(t.toSql())) .collect(toList()))); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTables.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTables.java index 2b8e5e8527c31..3bc69b2958ee5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTables.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTables.java @@ -33,17 +33,21 @@ public class SysTables extends Command { private final LikePattern pattern; private final LikePattern clusterPattern; private final EnumSet types; + // flag indicating whether tables are reported as `TABLE` or `BASE TABLE` + private final boolean legacyTableTypes; - public SysTables(Location location, LikePattern clusterPattern, LikePattern pattern, EnumSet types) { + public SysTables(Location location, LikePattern clusterPattern, LikePattern pattern, EnumSet types, + boolean legacyTableTypes) { super(location); this.clusterPattern = clusterPattern; this.pattern = pattern; this.types = types; + this.legacyTableTypes = legacyTableTypes; } @Override protected NodeInfo info() { - return NodeInfo.create(this, SysTables::new, clusterPattern, pattern, types); + return NodeInfo.create(this, SysTables::new, clusterPattern, pattern, types, legacyTableTypes); } @Override @@ -111,7 +115,7 @@ public final void execute(SqlSession session, ActionListener liste .map(t -> asList(cluster, EMPTY, t.name(), - t.type().toSql(), + legacyName(t.type()), EMPTY, null, null, @@ -122,6 +126,10 @@ public final void execute(SqlSession session, ActionListener liste , listener::onFailure)); } + private String legacyName(IndexType indexType) { + return legacyTableTypes && indexType == IndexType.INDEX ? "TABLE" : indexType.toSql(); + } + @Override public int hashCode() { return Objects.hash(clusterPattern, pattern, types); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java index 508ffef530573..ab40b076fac85 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.tree.NodeInfo; import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypes; import java.sql.DatabaseMetaData; import java.util.Comparator; @@ -67,9 +68,10 @@ public List output() { public final void execute(SqlSession session, ActionListener listener) { List> rows = Stream.of(DataType.values()) // sort by SQL int type (that's what the JDBC/ODBC specs want) - .sorted(Comparator.comparing(t -> t.jdbcType)) + .sorted(Comparator.comparing(t -> t.jdbcType.getVendorTypeNumber())) .map(t -> asList(t.esType.toUpperCase(Locale.ROOT), t.jdbcType.getVendorTypeNumber(), + //https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/column-size?view=sql-server-2017 t.defaultPrecision, "'", "'", @@ -83,16 +85,17 @@ public final void execute(SqlSession session, ActionListener liste // only numerics are signed !t.isSigned(), //no fixed precision scale SQL_FALSE - false, - null, - null, - null, + Boolean.FALSE, + // not auto-incremented + Boolean.FALSE, null, + DataTypes.metaSqlMinimumScale(t), + DataTypes.metaSqlMaximumScale(t), // SQL_DATA_TYPE - ODBC wants this to be not null - 0, - null, + DataTypes.metaSqlDataType(t), + DataTypes.metaSqlDateTimeSub(t), // Radix - t.isInteger ? Integer.valueOf(10) : (t.isRational ? Integer.valueOf(2) : null), + DataTypes.metaSqlRadix(t), null )) .collect(toList()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java index c0f4947bb88b3..c0bc9b6e52908 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java @@ -31,7 +31,7 @@ */ public abstract class DataTypeConversion { - private static final DateTimeFormatter UTC_DATE_FORMATTER = ISODateTimeFormat.dateTimeNoMillis().withZoneUTC(); + private static final DateTimeFormatter UTC_DATE_FORMATTER = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC(); /** * Returns the type compatible with both left and right types diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypes.java index c2b40656ba294..6fc7f95bef71e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypes.java @@ -51,4 +51,71 @@ public static DataType fromJava(Object value) { } throw new SqlIllegalArgumentException("No idea what's the DataType for {}", value.getClass()); } -} + + // + // Metadata methods, mainly for ODBC. + // As these are fairly obscure and limited in use, there is no point to promote them as a full type methods + // hence why they appear here as utility methods. + // + + // https://docs.microsoft.com/en-us/sql/relational-databases/native-client-odbc-date-time/metadata-catalog + // https://github.com/elastic/elasticsearch/issues/30386 + public static Integer metaSqlDataType(DataType t) { + if (t == DataType.DATE) { + // ODBC SQL_DATETME + return Integer.valueOf(9); + } + // this is safe since the vendor SQL types are short despite the return value + return t.jdbcType.getVendorTypeNumber(); + } + + // https://github.com/elastic/elasticsearch/issues/30386 + // https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlgettypeinfo-function?view=sql-server-2017 + public static Integer metaSqlDateTimeSub(DataType t) { + if (t == DataType.DATE) { + // ODBC SQL_CODE_TIMESTAMP + return Integer.valueOf(3); + } + // ODBC null + return 0; + } + + // https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/decimal-digits?view=sql-server-2017 + public static Short metaSqlMinimumScale(DataType t) { + // TODO: return info for HALF/SCALED_FLOATS (should be based on field not type) + if (t == DataType.DATE) { + return Short.valueOf((short) 3); + } + if (t.isInteger) { + return Short.valueOf((short) 0); + } + // minimum scale? + if (t.isRational) { + return Short.valueOf((short) 0); + } + return null; + } + + public static Short metaSqlMaximumScale(DataType t) { + // TODO: return info for HALF/SCALED_FLOATS (should be based on field not type) + if (t == DataType.DATE) { + return Short.valueOf((short) 3); + } + if (t.isInteger) { + return Short.valueOf((short) 0); + } + if (t.isRational) { + return Short.valueOf((short) t.defaultPrecision); + } + return null; + } + + // https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlgettypeinfo-function?view=sql-server-2017 + public static Integer metaSqlRadix(DataType t) { + // RADIX - Determines how numbers returned by COLUMN_SIZE and DECIMAL_DIGITS should be interpreted. + // 10 means they represent the number of decimal digits allowed for the column. + // 2 means they represent the number of bits allowed for the column. + // null means radix is not applicable for the given type. + return t.isInteger ? Integer.valueOf(10) : (t.isRational ? Integer.valueOf(2) : null); + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DateEsField.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DateEsField.java index b9737fbba608f..04926db5407f5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DateEsField.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DateEsField.java @@ -25,13 +25,6 @@ public DateEsField(String name, Map properties, boolean hasDocV this.formats = CollectionUtils.isEmpty(formats) ? DEFAULT_FORMAT : Arrays.asList(formats); } - @Override - public int getPrecision() { - // same as Long - // TODO: based this on format string - return 19; - } - public List getFormats() { return formats; } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index bddddc6941cbb..0b82530022386 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -38,6 +38,13 @@ public void testSysColumns() { assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); + row = rows.get(4); + assertEquals("date", name(row)); + assertEquals(Types.TIMESTAMP, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(24, precision(row)); + assertEquals(8, bufferLength(row)); + row = rows.get(7); assertEquals("some.dotted", name(row)); assertEquals(Types.STRUCT, sqlType(row)); @@ -59,6 +66,10 @@ private static Object sqlType(List list) { return list.get(4); } + private static Object precision(List list) { + return list.get(6); + } + private static Object bufferLength(List list) { return list.get(7); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java index ac72bcba4d647..27ed27413110f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java @@ -57,8 +57,8 @@ private Tuple sql(String sql) { public void testSysTypes() throws Exception { Command cmd = sql("SYS TYPES").v1(); - List names = asList("BYTE", "SHORT", "INTEGER", "LONG", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE", "KEYWORD", "TEXT", - "DATE", "BINARY", "NULL", "UNSUPPORTED", "OBJECT", "NESTED", "BOOLEAN"); + List names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE", + "KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "OBJECT", "NESTED"); cmd.execute(null, ActionListener.wrap(r -> { assertEquals(19, r.columnCount()); @@ -68,6 +68,8 @@ public void testSysTypes() throws Exception { assertFalse(r.column(9, Boolean.class)); // make sure precision is returned as boolean (not int) assertFalse(r.column(10, Boolean.class)); + // no auto-increment + assertFalse(r.column(11, Boolean.class)); for (int i = 0; i < r.size(); i++) { assertEquals(names.get(i), r.column(0)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTableTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTableTypesTests.java index 956273b9aae2d..291f9ee244e5f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTableTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTableTypesTests.java @@ -41,9 +41,9 @@ public void testSysCatalogs() throws Exception { sql.v1().execute(sql.v2(), ActionListener.wrap(r -> { assertEquals(2, r.size()); - assertEquals("BASE TABLE", r.column(0)); - r.advanceRow(); assertEquals("ALIAS", r.column(0)); + r.advanceRow(); + assertEquals("BASE TABLE", r.column(0)); }, ex -> fail(ex.getMessage()))); } } \ No newline at end of file diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index fe36095641a60..c7c9ab449c663 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -105,6 +105,23 @@ public void testSysTablesOnlyIndices() throws Exception { }, index); } + public void testSysTablesOnlyIndicesInLegacyMode() throws Exception { + executeCommand("SYS TABLES LIKE 'test' TYPE 'TABLE'", r -> { + assertEquals(1, r.size()); + assertEquals("test", r.column(2)); + assertEquals("TABLE", r.column(3)); + }, index); + + } + + public void testSysTablesOnlyIndicesLegacyModeParameterized() throws Exception { + executeCommand("SYS TABLES LIKE 'test' TYPE ?", asList(param("TABLE")), r -> { + assertEquals(1, r.size()); + assertEquals("test", r.column(2)); + assertEquals("TABLE", r.column(3)); + }, index); + } + public void testSysTablesOnlyIndicesParameterized() throws Exception { executeCommand("SYS TABLES LIKE 'test' TYPE ?", asList(param("ALIAS")), r -> { assertEquals(1, r.size()); @@ -131,6 +148,18 @@ public void testSysTablesOnlyIndicesAndAliasesParameterized() throws Exception { }, index, alias); } + public void testSysTablesOnlyIndicesLegacyAndAliasesParameterized() throws Exception { + List params = asList(param("ALIAS"), param("TABLE")); + executeCommand("SYS TABLES LIKE 'test' TYPE ?, ?", params, r -> { + assertEquals(2, r.size()); + assertEquals("test", r.column(2)); + assertEquals("TABLE", r.column(3)); + assertTrue(r.advanceRow()); + assertEquals("alias", r.column(2)); + assertEquals("ALIAS", r.column(3)); + }, index, alias); + } + public void testSysTablesWithCatalogOnlyAliases() throws Exception { executeCommand("SYS TABLES CATALOG LIKE '%' LIKE 'test' TYPE 'ALIAS'", r -> { assertEquals(1, r.size()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java index a6a322b31838f..8f5477f1951e9 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java @@ -82,10 +82,15 @@ public void testConversionToDate() { Conversion conversion = DataTypeConversion.conversionFor(DataType.KEYWORD, to); assertNull(conversion.convert(null)); - // TODO we'd like to be able to optionally parse millis here I think.... assertEquals(new DateTime(1000L, DateTimeZone.UTC), conversion.convert("1970-01-01T00:00:01Z")); assertEquals(new DateTime(1483228800000L, DateTimeZone.UTC), conversion.convert("2017-01-01T00:00:00Z")); assertEquals(new DateTime(18000000L, DateTimeZone.UTC), conversion.convert("1970-01-01T00:00:00-05:00")); + + // double check back and forth conversion + DateTime dt = DateTime.now(DateTimeZone.UTC); + Conversion forward = DataTypeConversion.conversionFor(DataType.DATE, DataType.KEYWORD); + Conversion back = DataTypeConversion.conversionFor(DataType.KEYWORD, DataType.DATE); + assertEquals(dt, back.convert(forward.convert(dt))); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); assertEquals("cannot cast [0xff] to [Date]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypesTests.java new file mode 100644 index 0000000000000..0a34c697bdf64 --- /dev/null +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypesTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.type; + +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.xpack.sql.type.DataType.DATE; +import static org.elasticsearch.xpack.sql.type.DataType.FLOAT; +import static org.elasticsearch.xpack.sql.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.sql.type.DataType.LONG; +import static org.elasticsearch.xpack.sql.type.DataTypes.metaSqlDataType; +import static org.elasticsearch.xpack.sql.type.DataTypes.metaSqlDateTimeSub; +import static org.elasticsearch.xpack.sql.type.DataTypes.metaSqlMaximumScale; +import static org.elasticsearch.xpack.sql.type.DataTypes.metaSqlMinimumScale; +import static org.elasticsearch.xpack.sql.type.DataTypes.metaSqlRadix; + +public class DataTypesTests extends ESTestCase { + + public void testMetaDataType() { + assertEquals(Integer.valueOf(9), metaSqlDataType(DATE)); + DataType t = randomDataTypeNoDate(); + assertEquals(t.jdbcType.getVendorTypeNumber(), metaSqlDataType(t)); + } + + public void testMetaDateTypeSub() { + assertEquals(Integer.valueOf(3), metaSqlDateTimeSub(DATE)); + assertEquals(Integer.valueOf(0), metaSqlDateTimeSub(randomDataTypeNoDate())); + } + + public void testMetaMinimumScale() { + assertEquals(Short.valueOf((short) 3), metaSqlMinimumScale(DATE)); + assertEquals(Short.valueOf((short) 0), metaSqlMinimumScale(LONG)); + assertEquals(Short.valueOf((short) 0), metaSqlMinimumScale(FLOAT)); + assertNull(metaSqlMinimumScale(KEYWORD)); + } + + public void testMetaMaximumScale() { + assertEquals(Short.valueOf((short) 3), metaSqlMaximumScale(DATE)); + assertEquals(Short.valueOf((short) 0), metaSqlMaximumScale(LONG)); + assertEquals(Short.valueOf((short) FLOAT.defaultPrecision), metaSqlMaximumScale(FLOAT)); + assertNull(metaSqlMaximumScale(KEYWORD)); + } + + public void testMetaRadix() { + assertNull(metaSqlRadix(DATE)); + assertNull(metaSqlRadix(KEYWORD)); + assertEquals(Integer.valueOf(10), metaSqlRadix(LONG)); + assertEquals(Integer.valueOf(2), metaSqlRadix(FLOAT)); + } + + private DataType randomDataTypeNoDate() { + return randomValueOtherThan(DataType.DATE, () -> randomFrom(DataType.values())); + } +} + diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java index c5e82123d7b8b..891b11ba70bb0 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java @@ -82,7 +82,7 @@ public void testDateField() { EsField field = mapping.get("date"); assertThat(field.getDataType(), is(DATE)); assertThat(field.hasDocValues(), is(true)); - assertThat(field.getPrecision(), is(19)); + assertThat(field.getPrecision(), is(24)); DateEsField dfield = (DateEsField) field; List formats = dfield.getFormats(); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml index ea545da5f639c..1a587c47fd573 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml @@ -88,7 +88,24 @@ setup: "description": "second", "latest_record_time_stamp": "2016-06-01T00:00:00Z", "latest_result_time_stamp": "2016-06-01T00:00:00Z", - "snapshot_doc_count": 3 + "snapshot_doc_count": 3, + "model_size_stats": { + "job_id" : "delete-model-snapshot", + "result_type" : "model_size_stats", + "model_bytes" : 0, + "total_by_field_count" : 101, + "total_over_field_count" : 0, + "total_partition_field_count" : 0, + "bucket_allocation_failures_count" : 0, + "memory_status" : "ok", + "log_time" : 1495808248662, + "timestamp" : 1495808248662 + }, + "quantiles": { + "job_id": "delete-model-snapshot", + "timestamp": 1495808248662, + "quantile_state": "quantiles-1" + } } - do: @@ -106,12 +123,10 @@ setup: - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - xpack.ml.update_job: + xpack.ml.revert_model_snapshot: job_id: delete-model-snapshot - body: > - { - "model_snapshot_id": "active-snapshot" - } + snapshot_id: "active-snapshot" + --- "Test delete snapshot missing snapshotId": diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/stats/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/stats/10_basic.yml index 6fa66667e2641..554e339687ba4 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/stats/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/stats/10_basic.yml @@ -1,13 +1,17 @@ --- "Test watcher stats output": - + - skip: + version: "all" + reason: "@AwaitsFix: https://github.com/elastic/elasticsearch/issues/30298" - do: {xpack.watcher.stats: {}} - match: { "manually_stopped": false } - match: { "stats.0.watcher_state": "started" } --- "Test watcher stats supports emit_stacktraces parameter": - + - skip: + version: "all" + reason: "@AwaitsFix: https://github.com/elastic/elasticsearch/issues/30298" - do: xpack.watcher.stats: metric: "all" diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java index ef5c3acc3d238..9f1fb95ed4835 100644 --- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java +++ b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.upgrade; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.search.SearchResponse; @@ -30,6 +31,7 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.core.IsEqual.equalTo; +@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30430") public class IndexUpgradeIT extends IndexUpgradeIntegTestCase { @Before diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java index 7d9e91384e515..3754cc440eb64 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java @@ -511,7 +511,7 @@ public interface Field { * @param params The ToXContentParams from the parent write * @param excludeField The field to exclude * @return A bytearrayinputstream that contains the serialized request - * @throws IOException + * @throws IOException if an IOException is triggered in the underlying toXContent method */ public static InputStream filterToXContent(HttpRequest request, XContent xContent, ToXContent.Params params, String excludeField) throws IOException { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/TriggerEngine.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/TriggerEngine.java index f370847aca965..ec63a68d3cae5 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/TriggerEngine.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/TriggerEngine.java @@ -37,11 +37,6 @@ public interface TriggerEngine { */ void pauseExecution(); - /** - * Returns the number of active jobs currently in this trigger engine implementation - */ - int getJobCount(); - /** * Removes the job associated with the given name from this trigger engine. * diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/manual/ManualTriggerEngine.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/manual/ManualTriggerEngine.java index 386e28501210e..2d44434206d76 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/manual/ManualTriggerEngine.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/manual/ManualTriggerEngine.java @@ -52,11 +52,6 @@ public void add(Watch job) { public void pauseExecution() { } - @Override - public int getJobCount() { - return 0; - } - @Override public boolean remove(String jobId) { return false; diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java index de8ab1d1f4bc6..05aa7cf302817 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java @@ -77,11 +77,6 @@ public void add(Watch watch) { schedules.put(watch.id(), new ActiveSchedule(watch.id(), trigger.getSchedule(), clock.millis())); } - @Override - public int getJobCount() { - return schedules.size(); - } - @Override public boolean remove(String jobId) { return schedules.remove(jobId) != null; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java index 57fe40f67b4dd..58f5c8f4a26b0 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java @@ -71,11 +71,6 @@ public void pauseExecution() { watches.clear(); } - @Override - public int getJobCount() { - return watches.size(); - } - @Override public boolean remove(String jobId) { return watches.remove(jobId) != null; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java index 572a361d25210..df96a802166e2 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java @@ -671,11 +671,6 @@ public void add(Watch watch) { public void pauseExecution() { } - @Override - public int getJobCount() { - return 0; - } - @Override public boolean remove(String jobId) { return false; diff --git a/x-pack/qa/openldap-tests/build.gradle b/x-pack/qa/openldap-tests/build.gradle index 6aef9fa28921f..24976ab6113ab 100644 --- a/x-pack/qa/openldap-tests/build.gradle +++ b/x-pack/qa/openldap-tests/build.gradle @@ -14,7 +14,7 @@ task openLdapFixture { dependsOn "vagrantCheckVersion", "virtualboxCheckVersion", idpFixtureProject.up } -String outputDir = "generated-resources/${project.name}" +String outputDir = "${project.buildDir}/generated-resources/${project.name}" task copyIdpTrust(type: Copy) { from idpFixtureProject.file('src/main/resources/certs/idptrust.jks'); into outputDir diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index a9e1ccba614ef..65b1a7c85dcb1 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField; import org.junit.Before; import java.io.IOException; @@ -67,7 +66,7 @@ protected Settings restClientSettings() { } protected Collection templatesToWaitFor() { - return Collections.singletonList(SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME); + return Collections.singletonList("security-index-template"); } @Before diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 251311fe8cd0a..ebb0d88991ff6 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -16,7 +16,7 @@ task idpFixture { dependsOn "vagrantCheckVersion", "virtualboxCheckVersion", idpFixtureProject.up } -String outputDir = "generated-resources/${project.name}" +String outputDir = "${project.buildDir}/generated-resources/${project.name}" task copyIdpCertificate(type: Copy) { from idpFixtureProject.file('src/main/resources/certs/ca.crt'); into outputDir diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlMultinodeIT.java b/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlMultinodeIT.java index efd426439e0ab..32dd60cfa2dce 100644 --- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlMultinodeIT.java +++ b/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlMultinodeIT.java @@ -8,6 +8,7 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; @@ -53,7 +54,7 @@ public void testIndexOnWrongNode() throws IOException { String firstHostName = null; String match = firstHost.getHostName() + ":" + firstHost.getPort(); - Map nodesInfo = responseToMap(client().performRequest("GET", "/_nodes")); + Map nodesInfo = responseToMap(client().performRequest(new Request("GET", "/_nodes"))); @SuppressWarnings("unchecked") Map nodes = (Map) nodesInfo.get("nodes"); for (Map.Entry node : nodes.entrySet()) { @@ -74,7 +75,9 @@ public void testIndexOnWrongNode() throws IOException { } index.endObject(); index.endObject(); - client().performRequest("PUT", "/test", emptyMap(), new StringEntity(Strings.toString(index), ContentType.APPLICATION_JSON)); + Request request = new Request("PUT", "/test"); + request.setJsonEntity(Strings.toString(index)); + client().performRequest(request); int documents = between(10, 100); createTestData(documents); @@ -84,6 +87,9 @@ public void testIndexOnWrongNode() throws IOException { } private void createTestData(int documents) throws UnsupportedCharsetException, IOException { + Request request = new Request("PUT", "/test/test/_bulk"); + request.addParameter("refresh", "true"); + StringBuilder bulk = new StringBuilder(); for (int i = 0; i < documents; i++) { int a = 3 * i; @@ -92,8 +98,9 @@ private void createTestData(int documents) throws UnsupportedCharsetException, I bulk.append("{\"index\":{\"_id\":\"" + i + "\"}\n"); bulk.append("{\"a\": " + a + ", \"b\": " + b + ", \"c\": " + c + "}\n"); } - client().performRequest("PUT", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + request.setJsonEntity(bulk.toString()); + + client().performRequest(request); } private Map responseToMap(Response response) throws IOException { @@ -108,14 +115,12 @@ private void assertCount(RestClient client, int count) throws IOException { expected.put("columns", singletonList(columnInfo(mode, "COUNT(1)", "long", JDBCType.BIGINT, 20))); expected.put("rows", singletonList(singletonList(count))); - Map params = new TreeMap<>(); - params.put("format", "json"); // JSON is easier to parse then a table - if (Strings.hasText(mode)) { - params.put("mode", mode); // JDBC or PLAIN mode + Request request = new Request("POST", "/_xpack/sql"); + if (false == mode.isEmpty()) { + request.addParameter("mode", mode); } - - Map actual = responseToMap(client.performRequest("POST", "/_xpack/sql", params, - new StringEntity("{\"query\": \"SELECT COUNT(*) FROM test\"}", ContentType.APPLICATION_JSON))); + request.setJsonEntity("{\"query\": \"SELECT COUNT(*) FROM test\"}"); + Map actual = responseToMap(client.performRequest(request)); if (false == expected.equals(actual)) { NotEqualMessageBuilder message = new NotEqualMessageBuilder(); diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java index 6ac1c2c11ea9b..5833ef6dae5a1 100644 --- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java +++ b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java @@ -10,6 +10,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Nullable; @@ -176,14 +177,15 @@ private static Map runSql(@Nullable String asUser, String mode, } private static Map runSql(@Nullable String asUser, String mode, HttpEntity entity) throws IOException { - Map params = new TreeMap<>(); - params.put("format", "json"); // JSON is easier to parse then a table - if (Strings.hasText(mode)) { - params.put("mode", mode); // JDBC or PLAIN mode + Request request = new Request("POST", "/_xpack/sql"); + if (false == mode.isEmpty()) { + request.addParameter("mode", mode); } - Header[] headers = asUser == null ? new Header[0] : new Header[] {new BasicHeader("es-security-runas-user", asUser)}; - Response response = client().performRequest("POST", "/_xpack/sql", params, entity, headers); - return toMap(response); + if (asUser != null) { + request.setHeaders(new BasicHeader("es-security-runas-user", asUser)); + } + request.setEntity(entity); + return toMap(client().performRequest(request)); } private static void assertResponse(Map expected, Map actual) { diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java index 205cd479dde1b..481e7a4f60f19 100644 --- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java +++ b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java @@ -11,6 +11,7 @@ import org.elasticsearch.SpecialPermission; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -41,7 +42,6 @@ import java.util.function.Function; import java.util.regex.Pattern; -import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; @@ -135,6 +135,9 @@ public void oneTimeSetup() throws Exception { * write the test data once. */ return; } + Request request = new Request("PUT", "/_bulk"); + request.addParameter("refresh", "true"); + StringBuilder bulk = new StringBuilder(); bulk.append("{\"index\":{\"_index\": \"test\", \"_type\": \"doc\", \"_id\":\"1\"}\n"); bulk.append("{\"a\": 1, \"b\": 2, \"c\": 3}\n"); @@ -142,8 +145,8 @@ public void oneTimeSetup() throws Exception { bulk.append("{\"a\": 4, \"b\": 5, \"c\": 6}\n"); bulk.append("{\"index\":{\"_index\": \"bort\", \"_type\": \"doc\", \"_id\":\"1\"}\n"); bulk.append("{\"a\": \"test\"}\n"); - client().performRequest("PUT", "/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + request.setJsonEntity(bulk.toString()); + client().performRequest(request); oneTimeSetup = true; } @@ -173,7 +176,7 @@ public void setInitialAuditLogOffset() { @AfterClass public static void wipeIndicesAfterTests() throws IOException { try { - adminClient().performRequest("DELETE", "*"); + adminClient().performRequest(new Request("DELETE", "*")); } catch (ResponseException e) { // 404 here just means we had no indexes if (e.getResponse().getStatusLine().getStatusCode() != 404) { @@ -472,13 +475,15 @@ public void testNoGetIndex() throws Exception { } protected static void createUser(String name, String role) throws IOException { - XContentBuilder user = JsonXContent.contentBuilder().prettyPrint().startObject(); { + Request request = new Request("PUT", "/_xpack/security/user/" + name); + XContentBuilder user = JsonXContent.contentBuilder().prettyPrint(); + user.startObject(); { user.field("password", "testpass"); user.field("roles", role); } user.endObject(); - client().performRequest("PUT", "/_xpack/security/user/" + name, emptyMap(), - new StringEntity(Strings.toString(user), ContentType.APPLICATION_JSON)); + request.setJsonEntity(Strings.toString(user)); + client().performRequest(request); } protected AuditLogAsserter createAuditLogAsserter() { diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/CliIntegrationTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/CliIntegrationTestCase.java index 63795edecf855..6adf37ff325e6 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/CliIntegrationTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/CliIntegrationTestCase.java @@ -5,9 +5,9 @@ */ package org.elasticsearch.xpack.qa.sql.cli; -import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Request; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -19,7 +19,6 @@ import java.io.IOException; -import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.assertNoSearchContexts; public abstract class CliIntegrationTestCase extends ESRestTestCase { @@ -60,11 +59,13 @@ protected SecurityConfig securityConfig() { } protected void index(String index, CheckedConsumer body) throws IOException { + Request request = new Request("PUT", "/" + index + "/doc/1"); + request.addParameter("refresh", "true"); XContentBuilder builder = JsonXContent.contentBuilder().startObject(); body.accept(builder); builder.endObject(); - HttpEntity doc = new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/" + index + "/doc/1", singletonMap("refresh", "true"), doc); + request.setJsonEntity(Strings.toString(builder)); + client().performRequest(request); } public String command(String command) throws IOException { diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java index 9a5d5b9c3eaca..f93ae339a820d 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java @@ -8,8 +8,7 @@ import java.io.IOException; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; - -import static java.util.Collections.emptyMap; +import org.elasticsearch.client.Request; import static org.hamcrest.Matchers.startsWith; @@ -41,7 +40,9 @@ public void testSelectFromMissingIndex() throws IOException { @Override public void testSelectFromIndexWithoutTypes() throws Exception { // Create an index without any types - client().performRequest("PUT", "/test", emptyMap(), new StringEntity("{}", ContentType.APPLICATION_JSON)); + Request request = new Request("PUT", "/test"); + request.setJsonEntity("{}"); + client().performRequest(request); assertFoundOneProblem(command("SELECT * FROM test")); assertEquals("line 1:15: [test] doesn't have any types so it is incompatible with sql" + END, readLine()); diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/FetchSizeTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/FetchSizeTestCase.java index dc34b9c1101c7..542e71ea1841e 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/FetchSizeTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/FetchSizeTestCase.java @@ -7,10 +7,10 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Request; import java.io.IOException; -import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.containsString; /** @@ -18,13 +18,16 @@ */ public abstract class FetchSizeTestCase extends CliIntegrationTestCase { public void testSelect() throws IOException { + Request request = new Request("PUT", "/test/doc/_bulk"); + request.addParameter("refresh", "true"); StringBuilder bulk = new StringBuilder(); for (int i = 0; i < 20; i++) { bulk.append("{\"index\":{}}\n"); bulk.append("{\"test_field\":" + i + "}\n"); } - client().performRequest("PUT", "/test/doc/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + request.setJsonEntity(bulk.toString()); + client().performRequest(request); + assertEquals("[?1l>[?1000l[?2004lfetch size set to [90m4[0m", command("fetch size = 4")); assertEquals("[?1l>[?1000l[?2004lfetch separator set to \"[90m -- fetch sep -- [0m\"", command("fetch separator = \" -- fetch sep -- \"")); diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java index 9137e2028aa50..f3fdd8e267ac3 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java @@ -8,6 +8,7 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Strings; @@ -55,6 +56,7 @@ private static void createString(String name, XContentBuilder builder) throws Ex .endObject(); } protected static void loadDatasetIntoEs(RestClient client, String index) throws Exception { + Request request = new Request("PUT", "/" + index); XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); createIndex.startObject("settings"); { @@ -91,11 +93,9 @@ protected static void loadDatasetIntoEs(RestClient client, String index) throws createIndex.endObject(); } createIndex.endObject().endObject(); - - client.performRequest("PUT", "/" + index, emptyMap(), new StringEntity(Strings.toString(createIndex), - ContentType.APPLICATION_JSON)); + request.setJsonEntity(Strings.toString(createIndex)); + client.performRequest(request); - Map deps = new LinkedHashMap<>(); csvToLines("departments", (titles, fields) -> deps.put(fields.get(0), fields.get(1))); @@ -119,6 +119,8 @@ protected static void loadDatasetIntoEs(RestClient client, String index) throws list.add(dep); }); + request = new Request("POST", "/" + index + "/emp/_bulk"); + request.addParameter("refresh", "true"); StringBuilder bulk = new StringBuilder(); csvToLines("employees", (titles, fields) -> { bulk.append("{\"index\":{}}\n"); @@ -146,17 +148,16 @@ protected static void loadDatasetIntoEs(RestClient client, String index) throws bulk.setLength(bulk.length() - 1); bulk.append("]"); } - + bulk.append("}\n"); }); - - client.performRequest("POST", "/" + index + "/emp/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + request.setJsonEntity(bulk.toString()); + client.performRequest(request); } protected static void makeAlias(RestClient client, String aliasName, String... indices) throws Exception { for (String index : indices) { - client.performRequest("POST", "/" + index + "/_alias/" + aliasName); + client.performRequest(new Request("POST", "/" + index + "/_alias/" + aliasName)); } } diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java index 0fffb0dac4c3b..ea6c5f165ee6f 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java @@ -9,8 +9,7 @@ import java.sql.SQLException; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; - -import static java.util.Collections.emptyMap; +import org.elasticsearch.client.Request; import static org.hamcrest.Matchers.startsWith; @@ -37,7 +36,9 @@ public void testSelectFromMissingIndex() throws SQLException { @Override public void testSelectFromIndexWithoutTypes() throws Exception { // Create an index without any types - client().performRequest("PUT", "/test", emptyMap(), new StringEntity("{}", ContentType.APPLICATION_JSON)); + Request request = new Request("PUT", "/test"); + request.setJsonEntity("{}"); + client().performRequest(request); try (Connection c = esJdbc()) { SQLException e = expectThrows(SQLException.class, () -> c.prepareStatement("SELECT * FROM test").executeQuery()); diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/FetchSizeTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/FetchSizeTestCase.java index de7cf465acacf..4d2487a0c03ff 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/FetchSizeTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/FetchSizeTestCase.java @@ -7,6 +7,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Request; import org.junit.Before; import java.io.IOException; @@ -15,7 +16,6 @@ import java.sql.SQLException; import java.sql.Statement; -import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.assertNoSearchContexts; /** @@ -25,13 +25,15 @@ public class FetchSizeTestCase extends JdbcIntegrationTestCase { @Before public void createTestIndex() throws IOException { + Request request = new Request("PUT", "/test/doc/_bulk"); + request.addParameter("refresh", "true"); StringBuilder bulk = new StringBuilder(); for (int i = 0; i < 20; i++) { bulk.append("{\"index\":{}}\n"); bulk.append("{\"test_field\":" + i + "}\n"); } - client().performRequest("PUT", "/test/doc/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + request.setJsonEntity(bulk.toString()); + client().performRequest(request); } /** diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java index aa5dc5c0ac2b6..fc0cd67efac14 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java @@ -9,6 +9,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -85,16 +86,18 @@ protected Connection useDataSource() throws SQLException { } public static void index(String index, CheckedConsumer body) throws IOException { + Request request = new Request("PUT", "/" + index + "/doc/1"); + request.addParameter("refresh", "true"); XContentBuilder builder = JsonXContent.contentBuilder().startObject(); body.accept(builder); builder.endObject(); - HttpEntity doc = new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/" + index + "/doc/1", singletonMap("refresh", "true"), doc); + request.setJsonEntity(Strings.toString(builder)); + client().performRequest(request); } protected String clusterName() { try { - String response = EntityUtils.toString(client().performRequest("GET", "/").getEntity()); + String response = EntityUtils.toString(client().performRequest(new Request("GET", "/")).getEntity()); return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false).get("cluster_name").toString(); } catch (IOException e) { throw new RuntimeException(e); diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SpecBaseIntegrationTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SpecBaseIntegrationTestCase.java index 5a589f94d28d4..d8ba1ade959ae 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SpecBaseIntegrationTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SpecBaseIntegrationTestCase.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.qa.sql.jdbc; import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; @@ -49,7 +50,7 @@ public SpecBaseIntegrationTestCase(String fileName, String groupName, String tes @Before public void setupTestDataIfNeeded() throws Exception { - if (client().performRequest("HEAD", "/test_emp").getStatusLine().getStatusCode() == 404) { + if (client().performRequest(new Request("HEAD", "/test_emp")).getStatusLine().getStatusCode() == 404) { DataLoader.loadDatasetIntoEs(client()); } } @@ -62,7 +63,7 @@ protected boolean preserveIndicesUponCompletion() { @AfterClass public static void wipeTestData() throws IOException { try { - adminClient().performRequest("DELETE", "/*"); + adminClient().performRequest(new Request("DELETE", "/*")); } catch (ResponseException e) { // 404 here just means we had no indexes if (e.getResponse().getStatusLine().getStatusCode() != 404) { diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java index 8062d7af497de..3019a00351c28 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java @@ -12,6 +12,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.CheckedSupplier; @@ -74,16 +75,19 @@ public void testBasicQuery() throws IOException { } public void testNextPage() throws IOException { + Request request = new Request("POST", "/test/test/_bulk"); + request.addParameter("refresh", "true"); String mode = randomMode(); StringBuilder bulk = new StringBuilder(); for (int i = 0; i < 20; i++) { bulk.append("{\"index\":{\"_id\":\"" + i + "\"}}\n"); bulk.append("{\"text\":\"text" + i + "\", \"number\":" + i + "}\n"); } - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + request.setJsonEntity(bulk.toString()); + client().performRequest(request); - String request = "{\"query\":\"" + String sqlRequest = + "{\"query\":\"" + " SELECT text, number, SQRT(number) AS s, SCORE()" + " FROM test" + " ORDER BY number, SCORE()\", " @@ -94,7 +98,7 @@ public void testNextPage() throws IOException { for (int i = 0; i < 20; i += 2) { Map response; if (i == 0) { - response = runSql(mode, new StringEntity(request, ContentType.APPLICATION_JSON)); + response = runSql(mode, new StringEntity(sqlRequest, ContentType.APPLICATION_JSON)); } else { response = runSql(mode, new StringEntity("{\"cursor\":\"" + cursor + "\"}", ContentType.APPLICATION_JSON)); @@ -138,12 +142,14 @@ public void testTimeZone() throws IOException { } public void testScoreWithFieldNamedScore() throws IOException { + Request request = new Request("POST", "/test/test/_bulk"); + request.addParameter("refresh", "true"); String mode = randomMode(); StringBuilder bulk = new StringBuilder(); bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); bulk.append("{\"name\":\"test\", \"score\":10}\n"); - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + request.setJsonEntity(bulk.toString()); + client().performRequest(request); Map expected = new HashMap<>(); expected.put("columns", Arrays.asList( @@ -209,7 +215,9 @@ public void testSelectFromMissingIndex() { @Override public void testSelectFromIndexWithoutTypes() throws Exception { // Create an index without any types - client().performRequest("PUT", "/test", emptyMap(), new StringEntity("{}", ContentType.APPLICATION_JSON)); + Request request = new Request("PUT", "/test"); + request.setJsonEntity("{}"); + client().performRequest(request); String mode = randomFrom("jdbc", "plain"); expectBadRequest(() -> runSql(mode, "SELECT * FROM test"), containsString("1:15: [test] doesn't have any types so it is incompatible with sql")); @@ -229,24 +237,9 @@ public void testSelectMissingFunction() throws Exception { containsString("1:8: Unknown function [missing]")); } - private void index(String... docs) throws IOException { - StringBuilder bulk = new StringBuilder(); - for (String doc : docs) { - bulk.append("{\"index\":{}\n"); - bulk.append(doc + "\n"); - } - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); - } - @Override public void testSelectProjectScoreInAggContext() throws Exception { - StringBuilder bulk = new StringBuilder(); - bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); - bulk.append("{\"foo\":1}\n"); - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); - + index("{\"foo\":1}"); expectBadRequest(() -> runSql(randomMode(), " SELECT foo, SCORE(), COUNT(*)" + " FROM test" @@ -256,12 +249,7 @@ public void testSelectProjectScoreInAggContext() throws Exception { @Override public void testSelectOrderByScoreInAggContext() throws Exception { - StringBuilder bulk = new StringBuilder(); - bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); - bulk.append("{\"foo\":1}\n"); - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); - + index("{\"foo\":1}"); expectBadRequest(() -> runSql(randomMode(), " SELECT foo, COUNT(*)" + " FROM test" @@ -272,36 +260,21 @@ public void testSelectOrderByScoreInAggContext() throws Exception { @Override public void testSelectGroupByScore() throws Exception { - StringBuilder bulk = new StringBuilder(); - bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); - bulk.append("{\"foo\":1}\n"); - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); - + index("{\"foo\":1}"); expectBadRequest(() -> runSql(randomMode(), "SELECT COUNT(*) FROM test GROUP BY SCORE()"), containsString("Cannot use [SCORE()] for grouping")); } @Override public void testSelectScoreSubField() throws Exception { - StringBuilder bulk = new StringBuilder(); - bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); - bulk.append("{\"foo\":1}\n"); - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); - + index("{\"foo\":1}"); expectBadRequest(() -> runSql(randomMode(), "SELECT SCORE().bar FROM test"), containsString("line 1:15: extraneous input '.' expecting {, ','")); } @Override public void testSelectScoreInScalar() throws Exception { - StringBuilder bulk = new StringBuilder(); - bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); - bulk.append("{\"foo\":1}\n"); - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); - + index("{\"foo\":1}"); expectBadRequest(() -> runSql(randomMode(), "SELECT SIN(SCORE()) FROM test"), containsString("line 1:12: [SCORE()] cannot be an argument to a function")); } @@ -340,37 +313,32 @@ private Map runSql(String mode, HttpEntity sql) throws IOExcepti } private Map runSql(String mode, HttpEntity sql, String suffix) throws IOException { - Map params = new TreeMap<>(); - params.put("error_trace", "true"); // Helps with debugging in case something crazy happens on the server. - params.put("pretty", "true"); // Improves error reporting readability + Request request = new Request("POST", "/_xpack/sql" + suffix); + request.addParameter("error_trace", "true"); // Helps with debugging in case something crazy happens on the server. + request.addParameter("pretty", "true"); // Improves error reporting readability if (randomBoolean()) { // We default to JSON but we force it randomly for extra coverage - params.put("format", "json"); + request.addParameter("format", "json"); } - if (Strings.hasText(mode)) { - params.put("mode", mode); // JDBC or PLAIN mode + if (false == mode.isEmpty()) { + request.addParameter("mode", mode); // JDBC or PLAIN mode } - Header[] headers = randomFrom( + request.setHeaders(randomFrom( new Header[] {}, new Header[] {new BasicHeader("Accept", "*/*")}, - new Header[] {new BasicHeader("Accpet", "application/json")}); - Response response = client().performRequest("POST", "/_xpack/sql" + suffix, params, sql); + new Header[] {new BasicHeader("Accpet", "application/json")})); + request.setEntity(sql); + Response response = client().performRequest(request); try (InputStream content = response.getEntity().getContent()) { return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); } } public void testBasicTranslateQuery() throws IOException { - StringBuilder bulk = new StringBuilder(); - bulk.append("{\"index\":{\"_id\":\"1\"}}\n"); - bulk.append("{\"test\":\"test\"}\n"); - bulk.append("{\"index\":{\"_id\":\"2\"}}\n"); - bulk.append("{\"test\":\"test\"}\n"); - client().performRequest("POST", "/test_translate/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + index("{\"test\":\"test\"}", "{\"test\":\"test\"}"); - Map response = runSql(randomMode(), "SELECT * FROM test_translate", "/translate/"); - assertEquals(response.get("size"), 1000); + Map response = runSql(randomMode(), "SELECT * FROM test", "/translate/"); + assertEquals(1000, response.get("size")); @SuppressWarnings("unchecked") Map source = (Map) response.get("_source"); assertNotNull(source); @@ -459,13 +427,12 @@ public void testBasicQueryText() throws IOException { } public void testNextPageText() throws IOException { - StringBuilder bulk = new StringBuilder(); - for (int i = 0; i < 20; i++) { - bulk.append("{\"index\":{\"_id\":\"" + i + "\"}}\n"); - bulk.append("{\"text\":\"text" + i + "\", \"number\":" + i + "}\n"); + int size = 20; + String[] docs = new String[size]; + for (int i = 0; i < size; i++) { + docs[i] = "{\"text\":\"text" + i + "\", \"number\":" + i + "}\n"; } - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + index(docs); String request = "{\"query\":\"SELECT text, number, number + 5 AS sum FROM test ORDER BY number\", \"fetch_size\":2}"; @@ -563,23 +530,33 @@ private Tuple runSqlAsText(String sql, String accept) throws IOE return runSqlAsText("", new StringEntity("{\"query\":\"" + sql + "\"}", ContentType.APPLICATION_JSON), accept); } + /** + * Run SQL as text using the {@code Accept} header to specify the format + * rather than the {@code format} parameter. + */ private Tuple runSqlAsText(String suffix, HttpEntity entity, String accept) throws IOException { - Response response = client().performRequest("POST", "/_xpack/sql" + suffix, singletonMap("error_trace", "true"), - entity, new BasicHeader("Accept", accept)); + Request request = new Request("POST", "/_xpack/sql" + suffix); + request.addParameter("error_trace", "true"); + request.setEntity(entity); + request.setHeaders(new BasicHeader("Accept", accept)); + Response response = client().performRequest(request); return new Tuple<>( Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8)), response.getHeader("Cursor") ); } + /** + * Run SQL as text using the {@code format} parameter to specify the format + * rather than an {@code Accept} header. + */ private Tuple runSqlAsTextFormat(String sql, String format) throws IOException { - StringEntity entity = new StringEntity("{\"query\":\"" + sql + "\"}", ContentType.APPLICATION_JSON); - - Map params = new HashMap<>(); - params.put("error_trace", "true"); - params.put("format", format); + Request request = new Request("POST", "/_xpack/sql"); + request.addParameter("error_trace", "true"); + request.addParameter("format", format); + request.setJsonEntity("{\"query\":\"" + sql + "\"}"); - Response response = client().performRequest("POST", "/_xpack/sql", params, entity); + Response response = client().performRequest(request); return new Tuple<>( Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8)), response.getHeader("Cursor") @@ -595,23 +572,14 @@ private void assertResponse(Map expected, Map ac } public static int getNumberOfSearchContexts(String index) throws IOException { - Response response = client().performRequest("GET", "/_stats/search"); - Map stats; - try (InputStream content = response.getEntity().getContent()) { - stats = XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); - } - return getOpenContexts(stats, index); + return getOpenContexts(searchStats(), index); } public static void assertNoSearchContexts() throws IOException { - Response response = client().performRequest("GET", "/_stats/search"); - Map stats; - try (InputStream content = response.getEntity().getContent()) { - stats = XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); - } + Map stats = searchStats(); @SuppressWarnings("unchecked") - Map indexStats = (Map) stats.get("indices"); - for (String index : indexStats.keySet()) { + Map indicesStats = (Map) stats.get("indices"); + for (String index : indicesStats.keySet()) { if (index.startsWith(".") == false) { // We are not interested in internal indices assertEquals(index + " should have no search contexts", 0, getOpenContexts(stats, index)); } @@ -619,12 +587,34 @@ public static void assertNoSearchContexts() throws IOException { } @SuppressWarnings("unchecked") - public static int getOpenContexts(Map indexStats, String index) { - return (int) ((Map) ((Map) ((Map) ((Map) - indexStats.get("indices")).get(index)).get("total")).get("search")).get("open_contexts"); + private static int getOpenContexts(Map stats, String index) { + stats = (Map) stats.get("indices"); + stats = (Map) stats.get(index); + stats = (Map) stats.get("total"); + stats = (Map) stats.get("search"); + return (Integer) stats.get("open_contexts"); + } + + private static Map searchStats() throws IOException { + Response response = client().performRequest(new Request("GET", "/_stats/search")); + try (InputStream content = response.getEntity().getContent()) { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); + } } public static String randomMode() { return randomFrom("", "jdbc", "plain"); } + + private void index(String... docs) throws IOException { + Request request = new Request("POST", "/test/test/_bulk"); + request.addParameter("refresh", "true"); + StringBuilder bulk = new StringBuilder(); + for (String doc : docs) { + bulk.append("{\"index\":{}\n"); + bulk.append(doc + "\n"); + } + request.setJsonEntity(bulk.toString()); + client().performRequest(request); + } } diff --git a/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_columns.sql b/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_columns.sql index 3d8cf4708945e..69c572f4ddd4e 100644 --- a/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_columns.sql +++ b/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_columns.sql @@ -25,26 +25,26 @@ CREATE TABLE mock ( ) AS SELECT null, 'test1', 'name', 12, 'TEXT', 0, 2147483647, null, null, 1, -- columnNullable - null, null, 12, null, 2147483647, 1, 'YES', null, null, null, null, 'NO', 'NO' + null, null, 12, 0, 2147483647, 1, 'YES', null, null, null, null, 'NO', 'NO' FROM DUAL UNION ALL SELECT null, 'test1', 'name.keyword', 12, 'KEYWORD', 0, 2147483647, null, null, 1, -- columnNullable - null, null, 12, null, 2147483647, 1, 'YES', null, null, null, null, 'NO', 'NO' + null, null, 12, 0, 2147483647, 1, 'YES', null, null, null, null, 'NO', 'NO' FROM DUAL UNION ALL -SELECT null, 'test2', 'date', 93, 'DATE', 20, 8, null, null, +SELECT null, 'test2', 'date', 93, 'DATE', 24, 8, null, null, 1, -- columnNullable - null, null, 93, null, null, 1, 'YES', null, null, null, null, 'NO', 'NO' + null, null, 9, 3, null, 1, 'YES', null, null, null, null, 'NO', 'NO' FROM DUAL UNION ALL SELECT null, 'test2', 'float', 7, 'FLOAT', 15, 4, null, 2, 1, -- columnNullable - null, null, 7, null, null, 2, 'YES', null, null, null, null, 'NO', 'NO' + null, null, 7, 0, null, 2, 'YES', null, null, null, null, 'NO', 'NO' FROM DUAL UNION ALL SELECT null, 'test2', 'number', -5, 'LONG', 20, 8, null, 10, 1, -- columnNullable - null, null, -5, null, null, 3, 'YES', null, null, null, null, 'NO', 'NO' + null, null, -5, 0, null, 3, 'YES', null, null, null, null, 'NO', 'NO' FROM DUAL ; diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java index 1d73d1f0d2979..11da59e44d6fe 100644 --- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java +++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java @@ -22,13 +22,13 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.xpack.core.security.SecurityLifecycleServiceField; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse; import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings; import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -198,7 +198,7 @@ public void cleanupSecurityIndex() throws Exception { @Override public Set excludeTemplates() { Set templates = Sets.newHashSet(super.excludeTemplates()); - templates.add(SecurityLifecycleServiceField.SECURITY_TEMPLATE_NAME); // don't remove the security index template + templates.add(SecurityIndexManager.SECURITY_TEMPLATE_NAME); // don't remove the security index template return templates; }