From fee000a37fb12f6cc92f10a8f5cc402badec696d Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Fri, 27 Apr 2018 09:24:46 -0700 Subject: [PATCH 1/6] [TEST] Redirect links to new locations (#30179) We had a number of awaitsFix links that weren't updated after the xpack merge. Where possible I changed the links to the new locations, but in some circumstances the original ticket was closed (suggesting the awaitsfix should be removed) or was otherwise unclear the status. --- .../xpack/monitoring/integration/MonitoringIT.java | 2 +- .../execution/search/extractor/FieldHitExtractorTests.java | 2 +- .../test/resources/rest-api-spec/test/upgrade/10_basic.yml | 2 +- .../xpack/watcher/test/integration/BootStrapTests.java | 4 ++-- .../src/test/java/org/elasticsearch/test/OpenLdapTests.java | 2 +- .../elasticsearch/smoketest/MonitoringWithWatcherRestIT.java | 2 +- .../org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java | 2 +- .../authc/ldap/ActiveDirectorySessionFactoryTests.java | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index fbe6c4658733e..a77f6bf24e99d 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -187,7 +187,7 @@ public void testMonitoringBulk() throws Exception { * This test waits for the monitoring service to collect monitoring documents and then checks that all expected documents * have been indexed with the expected information. */ - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/4150") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29880") @SuppressWarnings("unchecked") public void testMonitoringService() throws Exception { final boolean createAPMIndex = randomBoolean(); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java index 316bc4b2bf488..de36969898c20 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java @@ -49,7 +49,7 @@ protected FieldHitExtractor mutateInstance(FieldHitExtractor instance) throws IO return new FieldHitExtractor(instance.fieldName() + "mutated", true, instance.hitName()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/3082") + @AwaitsFix(bugUrl = "implement after we're sure of the InnerHitExtractor's implementation") public void testGetNested() throws IOException { fail("implement after we're sure of the InnerHitExtractor's implementation"); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/upgrade/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/upgrade/10_basic.yml index 1a21a4041ccd0..cfa75e0b4bdfd 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/upgrade/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/upgrade/10_basic.yml @@ -2,7 +2,7 @@ setup: - skip: version: "all" - reason: "AwaitsFix'ing, see x-pack-elasticsearch #4197" + reason: "AwaitsFix'ing, see https://github.com/elastic/elasticsearch/issues/29890" - do: xpack.license.post: body: > diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java index 1e3a9e43a1019..e47727f5d1085 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java @@ -145,7 +145,7 @@ public void testLoadMalformedWatchRecord() throws Exception { assertThat(response.getWatchesCount(), equalTo(1L)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1915") + @AwaitsFix(bugUrl = "Supposedly fixed; https://github.com/elastic/x-pack-elasticsearch/issues/1915") public void testLoadExistingWatchesUponStartup() throws Exception { stopWatcher(); @@ -226,7 +226,7 @@ public void testMixedTriggeredWatchLoading() throws Exception { assertSingleExecutionAndCompleteWatchHistory(numWatches, numRecords); } - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/3437") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29846") public void testTriggeredWatchLoading() throws Exception { createIndex("output"); client().prepareIndex("my-index", "foo", "bar") diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java index c6e10130db7c2..eced8a1b39ae8 100644 --- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java +++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java @@ -161,7 +161,7 @@ public void testCustomFilter() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/2849") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29758") public void testTcpTimeout() throws Exception { String groupSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; String userTemplate = "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java b/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java index 3d5c8e3f9f4a5..d89d558f02fae 100644 --- a/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java +++ b/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java @@ -31,7 +31,7 @@ import static org.hamcrest.Matchers.is; @TestLogging("org.elasticsearch.client:TRACE,tracer:TRACE") -@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/2920") +@AwaitsFix(bugUrl = "flaky tests") public class MonitoringWithWatcherRestIT extends ESRestTestCase { @After diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java index e970fcaa88a89..8062d7af497de 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java @@ -121,7 +121,7 @@ public void testNextPage() throws IOException { ContentType.APPLICATION_JSON))); } - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/2074") + @AwaitsFix(bugUrl = "Unclear status, https://github.com/elastic/x-pack-elasticsearch/issues/2074") public void testTimeZone() throws IOException { String mode = randomMode(); index("{\"test\":\"2017-07-27 00:00:00\"}", diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java index 2f1aa0f5eb573..a319578072def 100644 --- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java +++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java @@ -304,7 +304,7 @@ public void testStandardLdapConnection() throws Exception { } @SuppressWarnings("unchecked") - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/3369") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29840") public void testHandlingLdapReferralErrors() throws Exception { String groupSearchBase = "DC=ad,DC=test,DC=elasticsearch,DC=com"; String userTemplate = "CN={0},CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com"; From dd666599f7d3450496cdbdc29094c21792dc52a8 Mon Sep 17 00:00:00 2001 From: Sue Gallagher <36747279+Sue-Gallagher@users.noreply.github.com> Date: Fri, 27 Apr 2018 09:29:27 -0700 Subject: [PATCH 2/6] [DOCS] Added 'on a single shard' to description of max_thread_count. Closes 28518 (#29686) --- docs/reference/index-modules/merge.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/index-modules/merge.asciidoc b/docs/reference/index-modules/merge.asciidoc index 97db09ba656c7..cc0613ec2870d 100644 --- a/docs/reference/index-modules/merge.asciidoc +++ b/docs/reference/index-modules/merge.asciidoc @@ -23,7 +23,8 @@ The merge scheduler supports the following _dynamic_ setting: `index.merge.scheduler.max_thread_count`:: - The maximum number of threads that may be merging at once. Defaults to + The maximum number of threads on a single shard that may be merging at once. + Defaults to `Math.max(1, Math.min(4, Runtime.getRuntime().availableProcessors() / 2))` which works well for a good solid-state-disk (SSD). If your index is on spinning platter drives instead, decrease this to 1. From d633130e1bbc6ad3965b0c7d006dd62bef0b6629 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Fri, 27 Apr 2018 09:47:11 -0700 Subject: [PATCH 3/6] Convert FieldCapabilitiesResponse to a ToXContentObject. (#30182) --- .../action/fieldcaps/FieldCapabilitiesResponse.java | 9 +++++---- .../rest/action/RestFieldCapabilitiesAction.java | 13 +------------ .../fieldcaps/FieldCapabilitiesResponseTests.java | 4 +--- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index 5e2202ac073af..959b4e572b714 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; @@ -41,7 +41,7 @@ /** * Response for {@link FieldCapabilitiesRequest} requests. */ -public class FieldCapabilitiesResponse extends ActionResponse implements ToXContentFragment { +public class FieldCapabilitiesResponse extends ActionResponse implements ToXContentObject { private static final ParseField FIELDS_FIELD = new ParseField("fields"); private Map> responseMap; @@ -123,8 +123,9 @@ private static void writeField(StreamOutput out, @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(FIELDS_FIELD.getPreferredName(), responseMap); - return builder; + return builder.startObject() + .field(FIELDS_FIELD.getPreferredName(), responseMap) + .endObject(); } public static FieldCapabilitiesResponse fromXContent(XContentParser parser) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java index b2aac8d50ea08..4c477334265f6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java @@ -81,17 +81,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, fieldRequest.indicesOptions( IndicesOptions.fromRequest(request, fieldRequest.indicesOptions()) ); - return channel -> client.fieldCaps(fieldRequest, - new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(FieldCapabilitiesResponse response, - XContentBuilder builder) throws Exception { - RestStatus status = OK; - builder.startObject(); - response.toXContent(builder, request); - builder.endObject(); - return new BytesRestResponse(status, builder); - } - }); + return channel -> client.fieldCaps(fieldRequest, new RestToXContentListener<>(channel)); } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index c8bd5d5188b67..61556fd9b28ed 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -110,10 +110,8 @@ protected Predicate getRandomFieldsExcludeFilter() { public void testToXContent() throws IOException { FieldCapabilitiesResponse response = createSimpleResponse(); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON) - .startObject(); + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); response.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); String generatedResponse = BytesReference.bytes(builder).utf8ToString(); assertEquals(( From 912fbb2211e4d04d305491de6e875c3e44d9475a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 27 Apr 2018 14:04:37 -0400 Subject: [PATCH 4/6] Reindex: Fold "from old" tests into reindex module (#30142) This folds the `:qa:reindex-from-old` project into the `:modules:reindex` project. This should speed up the build marginally by removing a single clsuter start up at the cost of having to wait for old versions of Elasticsearch to start up when checking reindex's integration tests. Those don't take that long so this feels worth it. --- modules/reindex/build.gradle | 62 ++++++++++++ .../remote}/ReindexFromOldRemoteIT.java | 6 +- qa/reindex-from-old/build.gradle | 94 ------------------- 3 files changed, 67 insertions(+), 95 deletions(-) rename {qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest => modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote}/ReindexFromOldRemoteIT.java (95%) delete mode 100644 qa/reindex-from-old/build.gradle diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 479fe78cc8071..f34f4cf52e09c 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -17,6 +17,10 @@ * under the License. */ +import org.apache.tools.ant.taskdefs.condition.Os + +import static org.elasticsearch.gradle.BuildPlugin.getJavaHome + apply plugin: 'elasticsearch.test-with-dependencies' esplugin { @@ -60,3 +64,61 @@ thirdPartyAudit.excludes = [ 'org.apache.log.Hierarchy', 'org.apache.log.Logger', ] + +// Support for testing reindex-from-remote against old Elaticsearch versions +configurations { + oldesFixture + es2 + es1 + es090 +} + +dependencies { + oldesFixture project(':test:fixtures:old-elasticsearch') + /* Right now we just test against the latest version of each major we expect + * reindex-from-remote to work against. We could randomize the versions but + * that doesn't seem worth it at this point. */ + es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip' + es1 'org.elasticsearch:elasticsearch:1.7.6@zip' + es090 'org.elasticsearch:elasticsearch:0.90.13@zip' +} + +if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // we can't get the pid files in windows so we skip reindex-from-old + integTestRunner.systemProperty "tests.fromOld", "false" +} else { + integTestRunner.systemProperty "tests.fromOld", "true" + /* Set up tasks to unzip and run the old versions of ES before running the + * integration tests. */ + for (String version : ['2', '1', '090']) { + Task unzip = task("unzipEs${version}", type: Sync) { + Configuration oldEsDependency = configurations['es' + version] + dependsOn oldEsDependency + /* Use a closure here to delay resolution of the dependency until we need + * it */ + from { + oldEsDependency.collect { zipTree(it) } + } + into temporaryDir + } + Task fixture = task("oldEs${version}Fixture", + type: org.elasticsearch.gradle.test.AntFixture) { + dependsOn project.configurations.oldesFixture + dependsOn unzip + executable = new File(project.runtimeJavaHome, 'bin/java') + env 'CLASSPATH', "${ -> project.configurations.oldesFixture.asPath }" + env 'JAVA_HOME', getJavaHome(it, 7) + args 'oldes.OldElasticsearch', + baseDir, + unzip.temporaryDir, + version == '090' + } + integTest.dependsOn fixture + integTestRunner { + /* Use a closure on the string to delay evaluation until right before we + * run the integration tests so that we can be sure that the file is + * ready. */ + systemProperty "es${version}.port", "${ -> fixture.addressAndPort }" + } + } +} diff --git a/qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java similarity index 95% rename from qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java rename to modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java index 459aff3439710..5d359053a6668 100644 --- a/qa/reindex-from-old/src/test/java/org/elasticsearch/smoketest/ReindexFromOldRemoteIT.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.smoketest; +package org.elasticsearch.index.reindex.remote; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -27,6 +27,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Booleans; import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; @@ -38,6 +39,9 @@ public class ReindexFromOldRemoteIT extends ESRestTestCase { private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException { + boolean enabled = Booleans.parseBoolean(System.getProperty("tests.fromOld")); + assumeTrue("test is disabled, probably because this is windows", enabled); + int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName)); try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { try { diff --git a/qa/reindex-from-old/build.gradle b/qa/reindex-from-old/build.gradle deleted file mode 100644 index 8da714dd6278a..0000000000000 --- a/qa/reindex-from-old/build.gradle +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -description = """\ -Tests reindex-from-remote against some specific versions of -Elasticsearch prior to 5.0. Versions of Elasticsearch >= 5.0 -should be able to use the standard launching mechanism which -is more flexible and reliable. -""" - - -import org.apache.tools.ant.taskdefs.condition.Os - -import static org.elasticsearch.gradle.BuildPlugin.getJavaHome - -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-test' - -integTestCluster { - // Whitelist reindexing from the local node so we can test it. - setting 'reindex.remote.whitelist', '127.0.0.1:*' -} - -configurations { - oldesFixture - es2 - es1 - es090 -} - -dependencies { - oldesFixture project(':test:fixtures:old-elasticsearch') - /* Right now we just test against the latest version of each major we expect - * reindex-from-remote to work against. We could randomize the versions but - * that doesn't seem worth it at this point. */ - es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip' - es1 'org.elasticsearch:elasticsearch:1.7.6@zip' - es090 'org.elasticsearch:elasticsearch:0.90.13@zip' -} - -if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // we can't get the pid files in windows so we skip that - integTest.enabled = false -} else { - /* Set up tasks to unzip and run the old versions of ES before running the - * integration tests. */ - for (String version : ['2', '1', '090']) { - Task unzip = task("unzipEs${version}", type: Sync) { - Configuration oldEsDependency = configurations['es' + version] - dependsOn oldEsDependency - /* Use a closure here to delay resolution of the dependency until we need - * it */ - from { - oldEsDependency.collect { zipTree(it) } - } - into temporaryDir - } - Task fixture = task("oldEs${version}Fixture", - type: org.elasticsearch.gradle.test.AntFixture) { - dependsOn project.configurations.oldesFixture - dependsOn unzip - executable = new File(project.runtimeJavaHome, 'bin/java') - env 'CLASSPATH', "${ -> project.configurations.oldesFixture.asPath }" - env 'JAVA_HOME', getJavaHome(it, 7) - args 'oldes.OldElasticsearch', - baseDir, - unzip.temporaryDir, - version == '090' - } - integTest.dependsOn fixture - integTestRunner { - /* Use a closure on the string to delay evaluation until right before we - * run the integration tests so that we can be sure that the file is - * ready. */ - systemProperty "es${version}.port", "${ -> fixture.addressAndPort }" - } - } -} From f4ed9026983d4446ded243872f5876adf127c8d3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 27 Apr 2018 14:19:00 -0400 Subject: [PATCH 5/6] CCS: Drop http address from remote cluster info (#29568) They are expensive to fetch and no longer needed by Kibana so they *shouldn't* be needed by anyone else either. Closes #29207 --- docs/reference/cluster/remote-info.asciidoc | 3 - .../release-notes/7.0.0-alpha1.asciidoc | 4 + .../test/multi_cluster/20_info.yml | 5 -- .../remote/TransportRemoteInfoAction.java | 5 +- .../cluster/RestRemoteClusterInfoAction.java | 13 +-- .../transport/RemoteClusterConnection.java | 66 ++------------- .../transport/RemoteClusterService.java | 16 +--- .../transport/RemoteConnectionInfo.java | 67 ++++++++------- .../RemoteClusterConnectionTests.java | 83 +++---------------- .../test/multi_cluster/20_info.yml | 6 -- 10 files changed, 71 insertions(+), 197 deletions(-) diff --git a/docs/reference/cluster/remote-info.asciidoc b/docs/reference/cluster/remote-info.asciidoc index d044f4dcad221..3dfcc201e7ac4 100644 --- a/docs/reference/cluster/remote-info.asciidoc +++ b/docs/reference/cluster/remote-info.asciidoc @@ -19,9 +19,6 @@ the configured remote cluster alias. `seeds`:: The configured initial seed transport addresses of the remote cluster. -`http_addresses`:: - The published http addresses of all connected remote nodes. - `connected`:: True if there is at least one connection to the remote cluster. diff --git a/docs/reference/release-notes/7.0.0-alpha1.asciidoc b/docs/reference/release-notes/7.0.0-alpha1.asciidoc index 618d9e70dcb30..1cc328f16598b 100644 --- a/docs/reference/release-notes/7.0.0-alpha1.asciidoc +++ b/docs/reference/release-notes/7.0.0-alpha1.asciidoc @@ -10,5 +10,9 @@ The changes listed below have been released for the first time in Elasticsearch Core:: * Tribe node has been removed in favor of Cross-Cluster-Search +Cross-Cluster-Search:: +* `http_addresses` has been removed from the <> API + because it is expensive to fetch and no longer needed by Kibana. + Rest API:: * The Clear Cache API only supports `POST` as HTTP method diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml index 04086c2f2cb16..b4487e4fefee1 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml @@ -7,7 +7,6 @@ - match: { my_remote_cluster.num_nodes_connected: 1} - match: { my_remote_cluster.max_connections_per_cluster: 1} - match: { my_remote_cluster.initial_connect_timeout: "30s" } - - is_true: my_remote_cluster.http_addresses.0 --- "Add transient remote cluster based on the preset cluster and check remote info": @@ -38,9 +37,6 @@ - do: cluster.remote_info: {} - - set: { my_remote_cluster.http_addresses.0: remote_http } - - match: { test_remote_cluster.http_addresses.0: $remote_http } - - match: { test_remote_cluster.connected: true } - match: { my_remote_cluster.connected: true } @@ -132,4 +128,3 @@ transient: search.remote.remote1.seeds: null search.remote.remote1.skip_unavailable: null - diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java index 0410f920c8a9a..36974633559b6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java @@ -30,6 +30,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import static java.util.stream.Collectors.toList; + public final class TransportRemoteInfoAction extends HandledTransportAction { private final RemoteClusterService remoteClusterService; @@ -45,7 +47,6 @@ public TransportRemoteInfoAction(Settings settings, ThreadPool threadPool, Trans @Override protected void doExecute(RemoteInfoRequest remoteInfoRequest, ActionListener listener) { - remoteClusterService.getRemoteConnectionInfos(ActionListener.wrap(remoteConnectionInfos - -> listener.onResponse(new RemoteInfoResponse(remoteConnectionInfos)), listener::onFailure)); + listener.onResponse(new RemoteInfoResponse(remoteClusterService.getRemoteConnectionInfos().collect(toList()))); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java index 75baf8cecaaa5..c17be138df19a 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -50,16 +51,8 @@ public String getName() { } @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) - throws IOException { - return channel -> client.execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest(), - new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(RemoteInfoResponse response, XContentBuilder builder) throws Exception { - response.toXContent(builder, request); - return new BytesRestResponse(RestStatus.OK, builder); - } - }); + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + return channel -> client.execute(RemoteInfoAction.INSTANCE, new RemoteInfoRequest(), new RestToXContentListener<>(channel)); } @Override public boolean canTripCircuitBreaker() { diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index fb4586d201bd7..f24a1a928d50f 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -602,66 +603,13 @@ void addConnectedNode(DiscoveryNode node) { } /** - * Fetches connection info for this connection + * Get the information about remote nodes to be rendered on {@code _remote/info} requests. */ - public void getConnectionInfo(ActionListener listener) { - final Optional anyNode = connectedNodes.getAny(); - if (anyNode.isPresent() == false) { - // not connected we return immediately - RemoteConnectionInfo remoteConnectionStats = new RemoteConnectionInfo(clusterAlias, - Collections.emptyList(), Collections.emptyList(), maxNumRemoteConnections, 0, - RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings), skipUnavailable); - listener.onResponse(remoteConnectionStats); - } else { - NodesInfoRequest request = new NodesInfoRequest(); - request.clear(); - request.http(true); - - transportService.sendRequest(anyNode.get(), NodesInfoAction.NAME, request, new TransportResponseHandler() { - @Override - public NodesInfoResponse newInstance() { - return new NodesInfoResponse(); - } - - @Override - public void handleResponse(NodesInfoResponse response) { - Collection httpAddresses = new HashSet<>(); - for (NodeInfo info : response.getNodes()) { - if (connectedNodes.contains(info.getNode()) && info.getHttp() != null) { - httpAddresses.add(info.getHttp().getAddress().publishAddress()); - } - } - - if (httpAddresses.size() < maxNumRemoteConnections) { - // just in case non of the connected nodes have http enabled we get other http enabled nodes instead. - for (NodeInfo info : response.getNodes()) { - if (nodePredicate.test(info.getNode()) && info.getHttp() != null) { - httpAddresses.add(info.getHttp().getAddress().publishAddress()); - } - if (httpAddresses.size() == maxNumRemoteConnections) { - break; // once we have enough return... - } - } - } - RemoteConnectionInfo remoteConnectionInfo = new RemoteConnectionInfo(clusterAlias, - seedNodes.stream().map(DiscoveryNode::getAddress).collect(Collectors.toList()), new ArrayList<>(httpAddresses), - maxNumRemoteConnections, connectedNodes.size(), - RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings), skipUnavailable); - listener.onResponse(remoteConnectionInfo); - } - - @Override - public void handleException(TransportException exp) { - listener.onFailure(exp); - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - }); - } - + public RemoteConnectionInfo getConnectionInfo() { + List seedNodeAddresses = seedNodes.stream().map(DiscoveryNode::getAddress).collect(Collectors.toList()); + TimeValue initialConnectionTimeout = RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings); + return new RemoteConnectionInfo(clusterAlias, seedNodeAddresses, maxNumRemoteConnections, connectedNodes.size(), + initialConnectionTimeout, skipUnavailable); } int getNumNodesConnected() { diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index f454571301777..5de0d5e62dd07 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; -import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -42,7 +41,6 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -56,6 +54,7 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.common.settings.Setting.boolSetting; @@ -348,17 +347,8 @@ public void close() throws IOException { IOUtils.close(remoteClusters.values()); } - public void getRemoteConnectionInfos(ActionListener> listener) { - final Map remoteClusters = this.remoteClusters; - if (remoteClusters.isEmpty()) { - listener.onResponse(Collections.emptyList()); - } else { - final GroupedActionListener actionListener = new GroupedActionListener<>(listener, - remoteClusters.size(), Collections.emptyList()); - for (RemoteClusterConnection connection : remoteClusters.values()) { - connection.getConnectionInfo(actionListener); - } - } + public Stream getRemoteConnectionInfos() { + return remoteClusters.values().stream().map(RemoteClusterConnection::getConnectionInfo); } /** diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java index cb51f7edce570..60067e18573ad 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java @@ -27,17 +27,18 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; +import static java.util.Collections.emptyList; + import java.io.IOException; import java.util.List; import java.util.Objects; /** * This class encapsulates all remote cluster information to be rendered on - * _remote/info requests. + * {@code _remote/info} requests. */ public final class RemoteConnectionInfo implements ToXContentFragment, Writeable { final List seedNodes; - final List httpAddresses; final int connectionsPerCluster; final TimeValue initialConnectionTimeout; final int numNodesConnected; @@ -45,12 +46,10 @@ public final class RemoteConnectionInfo implements ToXContentFragment, Writeable final boolean skipUnavailable; RemoteConnectionInfo(String clusterAlias, List seedNodes, - List httpAddresses, int connectionsPerCluster, int numNodesConnected, TimeValue initialConnectionTimeout, boolean skipUnavailable) { this.clusterAlias = clusterAlias; this.seedNodes = seedNodes; - this.httpAddresses = httpAddresses; this.connectionsPerCluster = connectionsPerCluster; this.numNodesConnected = numNodesConnected; this.initialConnectionTimeout = initialConnectionTimeout; @@ -59,16 +58,45 @@ public final class RemoteConnectionInfo implements ToXContentFragment, Writeable public RemoteConnectionInfo(StreamInput input) throws IOException { seedNodes = input.readList(TransportAddress::new); - httpAddresses = input.readList(TransportAddress::new); + if (input.getVersion().before(Version.V_7_0_0_alpha1)) { + /* + * Versions before 7.0 sent the HTTP addresses of all nodes in the + * remote cluster here but it was expensive to fetch and we + * ultimately figured out how to do without it. So we removed it. + * + * We just throw any HTTP addresses received here on the floor + * because we don't need to do anything with them. + */ + input.readList(TransportAddress::new); + } connectionsPerCluster = input.readVInt(); initialConnectionTimeout = input.readTimeValue(); numNodesConnected = input.readVInt(); clusterAlias = input.readString(); - if (input.getVersion().onOrAfter(Version.V_6_1_0)) { - skipUnavailable = input.readBoolean(); - } else { - skipUnavailable = false; + skipUnavailable = input.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeList(seedNodes); + if (out.getVersion().before(Version.V_7_0_0_alpha1)) { + /* + * Versions before 7.0 sent the HTTP addresses of all nodes in the + * remote cluster here but it was expensive to fetch and we + * ultimately figured out how to do without it. So we removed it. + * + * When sending this request to a node that expects HTTP addresses + * here we pretend that we didn't find any. This *should* be fine + * because, after all, we haven't been using this information for + * a while. + */ + out.writeList(emptyList()); } + out.writeVInt(connectionsPerCluster); + out.writeTimeValue(initialConnectionTimeout); + out.writeVInt(numNodesConnected); + out.writeString(clusterAlias); + out.writeBoolean(skipUnavailable); } @Override @@ -80,11 +108,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(addr.toString()); } builder.endArray(); - builder.startArray("http_addresses"); - for (TransportAddress addr : httpAddresses) { - builder.value(addr.toString()); - } - builder.endArray(); builder.field("connected", numNodesConnected > 0); builder.field("num_nodes_connected", numNodesConnected); builder.field("max_connections_per_cluster", connectionsPerCluster); @@ -95,19 +118,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeList(seedNodes); - out.writeList(httpAddresses); - out.writeVInt(connectionsPerCluster); - out.writeTimeValue(initialConnectionTimeout); - out.writeVInt(numNodesConnected); - out.writeString(clusterAlias); - if (out.getVersion().onOrAfter(Version.V_6_1_0)) { - out.writeBoolean(skipUnavailable); - } - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -116,7 +126,6 @@ public boolean equals(Object o) { return connectionsPerCluster == that.connectionsPerCluster && numNodesConnected == that.numNodesConnected && Objects.equals(seedNodes, that.seedNodes) && - Objects.equals(httpAddresses, that.httpAddresses) && Objects.equals(initialConnectionTimeout, that.initialConnectionTimeout) && Objects.equals(clusterAlias, that.clusterAlias) && skipUnavailable == that.skipUnavailable; @@ -124,7 +133,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(seedNodes, httpAddresses, connectionsPerCluster, initialConnectionTimeout, + return Objects.hash(seedNodes, connectionsPerCluster, initialConnectionTimeout, numNodesConnected, clusterAlias, skipUnavailable); } } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 0d8a469981966..69096677664b3 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -42,6 +42,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; @@ -717,22 +718,6 @@ public void run() { } } - private static void installNodeStatsHandler(TransportService service, DiscoveryNode...nodes) { - service.registerRequestHandler(NodesInfoAction.NAME, NodesInfoRequest::new, ThreadPool.Names.SAME, false, false, - (request, channel) -> { - List nodeInfos = new ArrayList<>(); - int port = 80; - for (DiscoveryNode node : nodes) { - HttpInfo http = new HttpInfo(new BoundTransportAddress(new TransportAddress[]{node.getAddress()}, - new TransportAddress(node.getAddress().address().getAddress(), port++)), 100); - nodeInfos.add(new NodeInfo(node.getVersion(), Build.CURRENT, node, null, null, null, null, null, null, http, null, - null, null)); - } - channel.sendResponse(new NodesInfoResponse(ClusterName.DEFAULT, nodeInfos, Collections.emptyList())); - }); - - } - public void testGetConnectionInfo() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService transport1 = startTransport("seed_node", knownNodes, Version.CURRENT); @@ -753,34 +738,24 @@ public void testGetConnectionInfo() throws Exception { service.acceptIncomingRequests(); int maxNumConnections = randomIntBetween(1, 5); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes, service, maxNumConnections, n -> true)) { + seedNodes, service, maxNumConnections, n -> true)) { // test no nodes connected - RemoteConnectionInfo remoteConnectionInfo = assertSerialization(getRemoteConnectionInfo(connection)); + RemoteConnectionInfo remoteConnectionInfo = assertSerialization(connection.getConnectionInfo()); assertNotNull(remoteConnectionInfo); assertEquals(0, remoteConnectionInfo.numNodesConnected); - assertEquals(0, remoteConnectionInfo.seedNodes.size()); - assertEquals(0, remoteConnectionInfo.httpAddresses.size()); + assertEquals(3, remoteConnectionInfo.seedNodes.size()); assertEquals(maxNumConnections, remoteConnectionInfo.connectionsPerCluster); assertEquals("test-cluster", remoteConnectionInfo.clusterAlias); - updateSeedNodes(connection, seedNodes); - expectThrows(RemoteTransportException.class, () -> getRemoteConnectionInfo(connection)); - - for (MockTransportService s : Arrays.asList(transport1, transport2, transport3)) { - installNodeStatsHandler(s, node1, node2, node3); - } - remoteConnectionInfo = getRemoteConnectionInfo(connection); - remoteConnectionInfo = assertSerialization(remoteConnectionInfo); + // Connect some nodes + updateSeedNodes(connection, seedNodes); + remoteConnectionInfo = assertSerialization(connection.getConnectionInfo()); assertNotNull(remoteConnectionInfo); assertEquals(connection.getNumNodesConnected(), remoteConnectionInfo.numNodesConnected); assertEquals(Math.min(3, maxNumConnections), connection.getNumNodesConnected()); assertEquals(3, remoteConnectionInfo.seedNodes.size()); - assertEquals(remoteConnectionInfo.httpAddresses.size(), Math.min(3, maxNumConnections)); assertEquals(maxNumConnections, remoteConnectionInfo.connectionsPerCluster); assertEquals("test-cluster", remoteConnectionInfo.clusterAlias); - for (TransportAddress address : remoteConnectionInfo.httpAddresses) { - assertTrue("port range mismatch: " + address.getPort(), address.getPort() >= 80 && address.getPort() <= 90); - } } } } @@ -789,48 +764,41 @@ public void testGetConnectionInfo() throws Exception { public void testRemoteConnectionInfo() throws IOException { RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 80)), 4, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats); RemoteConnectionInfo stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 80)), 4, 4, TimeValue.timeValueMinutes(30), true); assertSerialization(stats1); assertNotEquals(stats, stats1); stats1 = new RemoteConnectionInfo("test_cluster_1", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 80)), 4, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats1); assertNotEquals(stats, stats1); stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 15)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 80)), 4, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats1); assertNotEquals(stats, stats1); stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 87)), 4, 3, TimeValue.timeValueMinutes(30), true); assertSerialization(stats1); assertNotEquals(stats, stats1); stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 80)), 4, 3, TimeValue.timeValueMinutes(325), true); assertSerialization(stats1); assertNotEquals(stats, stats1); stats1 = new RemoteConnectionInfo("test_cluster", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS, 80)), 5, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats1); assertNotEquals(stats, stats1); @@ -850,13 +818,14 @@ private static RemoteConnectionInfo assertSerialization(RemoteConnectionInfo inf } public void testRemoteConnectionInfoBwComp() throws IOException { - final Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_6_5, Version.V_6_0_0); + final Version version = VersionUtils.randomVersionBetween(random(), + Version.V_6_1_0, VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1)); RemoteConnectionInfo expected = new RemoteConnectionInfo("test_cluster", Collections.singletonList(new TransportAddress(TransportAddress.META_ADDRESS, 1)), - Collections.singletonList(new TransportAddress(TransportAddress.META_ADDRESS, 80)), 4, 4, new TimeValue(30, TimeUnit.MINUTES), false); - String encoded = "AQQAAAAABzAuMC4wLjAAAAABAQQAAAAABzAuMC4wLjAAAABQBDwEBAx0ZXN0X2NsdXN0ZXIAAAAAAAAAAAAAAA=="; + // This version was created using the serialization code in use from 6.1 but before 7.0 + String encoded = "AQQAAAAABzAuMC4wLjAAAAABAQQAAAAABzAuMC4wLjAAAABQBDwEBAx0ZXN0X2NsdXN0ZXIA"; final byte[] data = Base64.getDecoder().decode(encoded); try (StreamInput in = StreamInput.wrap(data)) { @@ -879,55 +848,29 @@ public void testRemoteConnectionInfoBwComp() throws IOException { public void testRenderConnectionInfoXContent() throws IOException { RemoteConnectionInfo stats = new RemoteConnectionInfo("test_cluster", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS,1)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS,80)), 4, 3, TimeValue.timeValueMinutes(30), true); stats = assertSerialization(stats); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); stats.toXContent(builder, null); builder.endObject(); - assertEquals("{\"test_cluster\":{\"seeds\":[\"0.0.0.0:1\"],\"http_addresses\":[\"0.0.0.0:80\"],\"connected\":true," + + assertEquals("{\"test_cluster\":{\"seeds\":[\"0.0.0.0:1\"],\"connected\":true," + "\"num_nodes_connected\":3,\"max_connections_per_cluster\":4,\"initial_connect_timeout\":\"30m\"," + "\"skip_unavailable\":true}}", Strings.toString(builder)); stats = new RemoteConnectionInfo("some_other_cluster", Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS,1), new TransportAddress(TransportAddress.META_ADDRESS,2)), - Arrays.asList(new TransportAddress(TransportAddress.META_ADDRESS,80), new TransportAddress(TransportAddress.META_ADDRESS,81)), 2, 0, TimeValue.timeValueSeconds(30), false); stats = assertSerialization(stats); builder = XContentFactory.jsonBuilder(); builder.startObject(); stats.toXContent(builder, null); builder.endObject(); - assertEquals("{\"some_other_cluster\":{\"seeds\":[\"0.0.0.0:1\",\"0.0.0.0:2\"],\"http_addresses\":[\"0.0.0.0:80\",\"0.0.0.0:81\"]," + assertEquals("{\"some_other_cluster\":{\"seeds\":[\"0.0.0.0:1\",\"0.0.0.0:2\"]," + "\"connected\":false,\"num_nodes_connected\":0,\"max_connections_per_cluster\":2,\"initial_connect_timeout\":\"30s\"," + "\"skip_unavailable\":false}}", Strings.toString(builder)); } - private RemoteConnectionInfo getRemoteConnectionInfo(RemoteClusterConnection connection) throws Exception { - AtomicReference statsRef = new AtomicReference<>(); - AtomicReference exceptionRef = new AtomicReference<>(); - CountDownLatch latch = new CountDownLatch(1); - connection.getConnectionInfo(new ActionListener() { - @Override - public void onResponse(RemoteConnectionInfo remoteConnectionInfo) { - statsRef.set(remoteConnectionInfo); - latch.countDown(); - } - - @Override - public void onFailure(Exception e) { - exceptionRef.set(e); - latch.countDown(); - } - }); - latch.await(); - if (exceptionRef.get() != null) { - throw exceptionRef.get(); - } - return statsRef.get(); - } - public void testEnsureConnected() throws IOException, InterruptedException { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); diff --git a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml index cf374db6a3b57..5ff92df69b863 100644 --- a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml +++ b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml @@ -41,8 +41,6 @@ teardown: - match: { my_remote_cluster.num_nodes_connected: 1} - match: { my_remote_cluster.max_connections_per_cluster: 1} - match: { my_remote_cluster.initial_connect_timeout: "30s" } - - is_true: my_remote_cluster.http_addresses.0 - --- "Add transient remote cluster based on the preset cluster and check remote info": @@ -70,9 +68,6 @@ teardown: - do: headers: { Authorization: "Basic am9lOnMza3JpdA==" } cluster.remote_info: {} - - set: { my_remote_cluster.http_addresses.0: remote_http } - - match: { test_remote_cluster.http_addresses.0: $remote_http } - - match: { test_remote_cluster.connected: true } - match: { my_remote_cluster.connected: true } @@ -87,4 +82,3 @@ teardown: - match: { my_remote_cluster.initial_connect_timeout: "30s" } - match: { test_remote_cluster.initial_connect_timeout: "30s" } - From 8401eac425d0440c0e860fb14cd112be2f8c01df Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 27 Apr 2018 15:01:42 -0400 Subject: [PATCH 6/6] Test: Switch painless test to 1 shard We *think* that #28600 is caused by warnings not being collected during one of the fan out phases of search but we're not 100% sure how this is happening. This commit drops the number of shards used for the test to 1 so there *isn't* a fan out phase. If this makes the issue go away we'll have more information. --- .../rest-api-spec/test/painless/50_script_doc_values.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml index ce8c03afec607..ede2927b992e0 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -3,6 +3,8 @@ setup: indices.create: index: test body: + settings: + number_of_shards: 1 mappings: test: properties: