From 95c2796d5fa767b6fa4084ebdda8e84fa7da9b83 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 08:48:36 -0500 Subject: [PATCH 01/43] Add 7.1 and 8.0 version constants to master This commit adds the 7.1 and 8.0 version constants to the master branch, and bumps the master branch to 8.0.0. --- buildSrc/version.properties | 2 +- docs/Versions.asciidoc | 4 ++-- server/src/main/java/org/elasticsearch/Version.java | 10 +++++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 118ab2f905f74..e6aadf6b82055 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,4 +1,4 @@ -elasticsearch = 7.0.0 +elasticsearch = 8.0.0 lucene = 8.0.0-snapshot-83f9835 # optional dependencies diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index 8a446bf037a12..d63cdf5033ecc 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,5 +1,5 @@ -:version: 7.0.0-alpha2 -:major-version: 7.x +:version: 8.0.0 +:major-version: 8.x :lucene_version: 8.0.0 :lucene_version_path: 8_0_0 :branch: master diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index e520d714bb931..c02062b2bbd63 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -124,7 +124,11 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_6_7_0 = new Version(V_6_7_0_ID, org.apache.lucene.util.Version.LUCENE_7_7_0); public static final int V_7_0_0_ID = 7000099; public static final Version V_7_0_0 = new Version(V_7_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); - public static final Version CURRENT = V_7_0_0; + public static final int V_7_1_0_ID = 7010099; + public static final Version V_7_1_0 = new Version(V_7_1_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); + public static final int V_8_0_0_ID = 8000099; + public static final Version V_8_0_0 = new Version(V_8_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); + public static final Version CURRENT = V_8_0_0; static { @@ -138,6 +142,10 @@ public static Version readVersion(StreamInput in) throws IOException { public static Version fromId(int id) { switch (id) { + case V_8_0_0_ID: + return V_8_0_0; + case V_7_1_0_ID: + return V_7_1_0; case V_7_0_0_ID: return V_7_0_0; case V_6_7_0_ID: From 78a4ce0ddf5d35bd27c49492396940775fe16b00 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 6 Feb 2019 18:28:51 +0200 Subject: [PATCH 02/43] Fix incorrect handling of version collections --- .../gradle/VersionCollection.java | 32 ++++++++++++------- .../gradle/VersionCollectionTests.java | 20 ++++++++++-- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java index e2af34dbabdc0..cf8eb1829abef 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java @@ -171,30 +171,38 @@ public UnreleasedVersionInfo unreleasedInfo(Version version) { } public void forPreviousUnreleased(Consumer consumer) { - getUnreleased().stream() + List collect = getUnreleased().stream() .filter(version -> version.equals(currentVersion) == false) - .forEach(version -> consumer.accept( - new UnreleasedVersionInfo( + .map(version -> new UnreleasedVersionInfo( version, getBranchFor(version), getGradleProjectNameFor(version) ) - )); + ) + .collect(Collectors.toList()); + + collect.forEach(uvi -> consumer.accept(uvi)); } private String getGradleProjectNameFor(Version version) { if (version.equals(currentVersion)) { throw new IllegalArgumentException("The Gradle project to build " + version + " is the current build."); } + Map> releasedMajorGroupedByMinor = getReleasedMajorGroupedByMinor(); if (version.getRevision() == 0) { - if (releasedMajorGroupedByMinor - .get(releasedMajorGroupedByMinor.keySet().stream().max(Integer::compareTo).orElse(0)) - .contains(version)) { - return "minor"; + List unreleasedStagedOrMinor = getUnreleased().stream() + .filter(v -> v.getRevision() == 0) + .collect(Collectors.toList()); + if (unreleasedStagedOrMinor.size() > 2) { + if (unreleasedStagedOrMinor.get(unreleasedStagedOrMinor.size() - 2).equals(version)) { + return "minor"; + } else{ + return "staged"; + } } else { - return "staged"; + return "minor"; } } else { if (releasedMajorGroupedByMinor @@ -239,8 +247,10 @@ public List getUnreleased() { unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 1)); if (groupByMinor.getOrDefault(greatestMinor - 1, emptyList()).size() == 1) { // we found that the previous minor is staged but not yet released - // in this case, the minor before that has a bugfix - unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2)); + // in this case, the minor before that has a bugfix, should there be such a minor + if (greatestMinor >= 2) { + unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2)); + } } } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java index d1b4e893ec6ad..4ea33b240e532 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java @@ -81,6 +81,9 @@ public class VersionCollectionTests extends GradleUnitTestCase { "6_0_0", "6_0_1", "6_1_0", "6_1_1", "6_1_2", "6_1_3", "6_1_4", "6_2_0", "6_2_1", "6_2_2", "6_2_3", "6_2_4", "6_3_0", "6_3_1", "6_3_2", "6_4_0", "6_4_1", "6_4_2" )); + sampleVersions.put("7.1.0", asList( + "7_1_0", "7_0_0", "6_7_0", "6_6_1", "6_6_0" + )); } @Test(expected = IllegalArgumentException.class) @@ -145,6 +148,11 @@ public void testWireCompatible() { singletonList("7.3.0"), getVersionCollection("8.0.0").getWireCompatible() ); + assertVersionsEquals( + asList("6.7.0", "7.0.0"), + getVersionCollection("7.1.0").getWireCompatible() + ); + } public void testWireCompatibleUnreleased() { @@ -171,6 +179,10 @@ public void testWireCompatibleUnreleased() { singletonList("7.3.0"), getVersionCollection("8.0.0").getUnreleasedWireCompatible() ); + assertVersionsEquals( + asList("6.7.0", "7.0.0"), + getVersionCollection("7.1.0").getWireCompatible() + ); } public void testIndexCompatible() { @@ -286,7 +298,7 @@ public void testGetBranch() { getVersionCollection("6.4.2") ); assertUnreleasedBranchNames( - asList("5.6", "6.4", "6.5"), + asList("5.6", "6.4", "6.x"), getVersionCollection("6.6.0") ); assertUnreleasedBranchNames( @@ -309,13 +321,17 @@ public void testGetGradleProjectName() { getVersionCollection("6.4.2") ); assertUnreleasedGradleProjectNames( - asList("maintenance", "bugfix", "staged"), + asList("maintenance", "bugfix", "minor"), getVersionCollection("6.6.0") ); assertUnreleasedGradleProjectNames( asList("bugfix", "staged", "minor"), getVersionCollection("8.0.0") ); + assertUnreleasedGradleProjectNames( + asList("staged", "minor"), + getVersionCollection("7.1.0") + ); } public void testCompareToAuthoritative() { From 7b07a62f08e80ac0d2a3d8796b501ba3cfedea8f Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 6 Feb 2019 19:09:07 +0200 Subject: [PATCH 03/43] Bump mastre version to 8.0.0 --- docs/Versions.asciidoc | 2 +- .../java/org/elasticsearch/node/InternalSettingsPreparer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index d63cdf5033ecc..89d0be877b839 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,4 +1,4 @@ -:version: 8.0.0 +:version: 8.0.0-alpha1 :major-version: 8.x :lucene_version: 8.0.0 :lucene_version_path: 8_0_0 diff --git a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java index e0157fff23cfd..623b763e84989 100644 --- a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java +++ b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java @@ -118,7 +118,7 @@ static void initializeSettings(final Settings.Builder output, final Settings inp */ private static void checkSettingsForTerminalDeprecation(final Settings.Builder output) throws SettingsException { // This method to be removed in 8.0.0, as it was deprecated in 6.0 and removed in 7.0 - assert Version.CURRENT.major != 8: "Logic pertaining to config driven prompting should be removed"; + // assert Version.CURRENT.major != 8: "Logic pertaining to config driven prompting should be removed"; for (String setting : output.keys()) { switch (output.get(setting)) { case SECRET_PROMPT_VALUE: From 49fadfad41bf65a5be5fd5602defd7dad703bc28 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 6 Feb 2019 19:39:55 +0200 Subject: [PATCH 04/43] Fix bwcBuilds for versions newer than 7.0.0 --- distribution/bwc/build.gradle | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 0ee597b449a41..ff111ecaa08ca 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -141,6 +141,12 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre extension += '.gz' } } + if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) { + classifier = "-amd64" + } + if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) { + classifier = "-x86_64" + } if (bwcVersion.onOrAfter('6.3.0')) { baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages' // add oss variant first From 05318172e13deadd6ea832118bd7111342d7d442 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 6 Feb 2019 14:55:04 -0500 Subject: [PATCH 05/43] move old versions to unreleased --- .../org/elasticsearch/test/VersionUtils.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index 84c480b8d510b..dbd786996bb15 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.collect.Tuple; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -52,7 +53,7 @@ static Tuple, List> resolveReleasedVersions(Version curre // this breaks b/c 5.x is still in version list but master doesn't care about it! //assert majorVersions.size() == 2; // TODO: remove oldVersions, we should only ever have 2 majors in Version - List oldVersions = majorVersions.getOrDefault((int)current.major - 2, Collections.emptyList()); + List> oldVersions = splitByMinor(majorVersions.getOrDefault((int)current.major - 2, Collections.emptyList())); List> previousMajor = splitByMinor(majorVersions.get((int)current.major - 1)); List> currentMajor = splitByMinor(majorVersions.get((int)current.major)); @@ -78,12 +79,21 @@ static Tuple, List> resolveReleasedVersions(Version curre moveLastToUnreleased(stableVersions, unreleasedVersions); } // remove the next bugfix - moveLastToUnreleased(stableVersions, unreleasedVersions); + if (stableVersions.isEmpty() == false) { + moveLastToUnreleased(stableVersions, unreleasedVersions); + } } - List releasedVersions = Stream.concat(oldVersions.stream(), - Stream.concat(previousMajor.stream(), currentMajor.stream()).flatMap(List::stream)) - .collect(Collectors.toList()); + // If none of the previous major was released, then the last minor and bugfix of the old version was not released either. + if (previousMajor.isEmpty()) { + assert currentMajor.isEmpty() : currentMajor; + // minor of the old version is being staged + moveLastToUnreleased(oldVersions, unreleasedVersions); + // bugix of the old version is also being staged + moveLastToUnreleased(oldVersions, unreleasedVersions); + } + List releasedVersions = Stream.of(oldVersions, previousMajor, currentMajor) + .flatMap(List::stream).flatMap(List::stream).collect(Collectors.toList()); Collections.sort(unreleasedVersions); // we add unreleased out of order, so need to sort here return new Tuple<>(Collections.unmodifiableList(releasedVersions), Collections.unmodifiableList(unreleasedVersions)); } From 633acbf4d51fd19d18fae8e0d2e7b8c89bbdbda9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 6 Feb 2019 18:24:59 +0000 Subject: [PATCH 06/43] Remove Zen1 BWC --- .../cluster/coordination/Coordinator.java | 9 - .../coordination/DiscoveryUpgradeService.java | 347 ---------------- .../PublicationTransportHandler.java | 22 -- .../common/settings/ClusterSettings.java | 7 +- .../elasticsearch/discovery/PeerFinder.java | 71 +--- .../DiscoveryUpgradeServiceTests.java | 38 -- .../cluster/coordination/Zen1IT.java | 373 ------------------ 7 files changed, 5 insertions(+), 862 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java delete mode 100644 server/src/test/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeServiceTests.java delete mode 100644 server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 6843534107178..9eed4600738b0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -127,7 +127,6 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private long maxTermSeen; private final Reconfigurator reconfigurator; private final ClusterBootstrapService clusterBootstrapService; - private final DiscoveryUpgradeService discoveryUpgradeService; private final LagDetector lagDetector; private final ClusterFormationFailureHelper clusterFormationFailureHelper; @@ -169,8 +168,6 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe this.reconfigurator = new Reconfigurator(settings, clusterSettings); this.clusterBootstrapService = new ClusterBootstrapService(settings, transportService, this::getFoundPeers, this::isInitialConfigurationSet, this::setInitialConfiguration); - this.discoveryUpgradeService = new DiscoveryUpgradeService(settings, transportService, - this::isInitialConfigurationSet, joinHelper, peerFinder::getFoundPeers, this::setInitialConfiguration); this.lagDetector = new LagDetector(settings, transportService.getThreadPool(), n -> removeNode(n, "lagging"), transportService::getLocalNode); this.clusterFormationFailureHelper = new ClusterFormationFailureHelper(settings, this::getClusterFormationState, @@ -508,10 +505,6 @@ void becomeCandidate(String method) { peerFinder.activate(coordinationState.get().getLastAcceptedState().nodes()); clusterFormationFailureHelper.start(); - if (getCurrentTerm() == ZEN1_BWC_TERM) { - discoveryUpgradeService.activate(lastKnownLeader, coordinationState.get().getLastAcceptedState()); - } - leaderChecker.setCurrentNodes(DiscoveryNodes.EMPTY_NODES); leaderChecker.updateLeader(null); @@ -543,7 +536,6 @@ void becomeLeader(String method) { lastKnownLeader = Optional.of(getLocalNode()); peerFinder.deactivate(getLocalNode()); - discoveryUpgradeService.deactivate(); clusterFormationFailureHelper.stop(); closePrevotingAndElectionScheduler(); preVoteCollector.update(getPreVoteResponse(), getLocalNode()); @@ -575,7 +567,6 @@ void becomeFollower(String method, DiscoveryNode leaderNode) { lastKnownLeader = Optional.of(leaderNode); peerFinder.deactivate(leaderNode); - discoveryUpgradeService.deactivate(); clusterFormationFailureHelper.stop(); closePrevotingAndElectionScheduler(); cancelActivePublication("become follower: " + method); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java deleted file mode 100644 index 3bf5fa225a934..0000000000000 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.cluster.coordination; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.CountDown; -import org.elasticsearch.discovery.zen.ElectMasterService; -import org.elasticsearch.discovery.zen.ElectMasterService.MasterCandidate; -import org.elasticsearch.discovery.zen.UnicastZenPing; -import org.elasticsearch.discovery.zen.UnicastZenPing.UnicastPingRequest; -import org.elasticsearch.discovery.zen.UnicastZenPing.UnicastPingResponse; -import org.elasticsearch.discovery.zen.ZenPing.PingResponse; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Names; -import org.elasticsearch.transport.TransportException; -import org.elasticsearch.transport.TransportRequestOptions; -import org.elasticsearch.transport.TransportResponseHandler; -import org.elasticsearch.transport.TransportService; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Optional; -import java.util.Set; -import java.util.function.BooleanSupplier; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import static java.lang.Math.max; -import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING; -import static org.elasticsearch.cluster.ClusterState.UNKNOWN_VERSION; -import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentSet; -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; -import static org.elasticsearch.discovery.zen.ZenDiscovery.PING_TIMEOUT_SETTING; - -/** - * Deals with rolling upgrades of the cluster coordination layer. In mixed clusters we prefer to elect the older nodes, but - * when the last old node shuts down then as long as there are enough new nodes we can assume that they form the whole cluster and - * define them as the initial configuration. - */ -public class DiscoveryUpgradeService { - - private static Logger logger = LogManager.getLogger(DiscoveryUpgradeService.class); - - // how long to wait after activation before attempting to join a master or perform a bootstrap upgrade - public static final Setting BWC_PING_TIMEOUT_SETTING = - Setting.timeSetting("discovery.zen.bwc_ping_timeout", - PING_TIMEOUT_SETTING, TimeValue.timeValueMillis(1), Setting.Property.NodeScope, Setting.Property.Deprecated); - - // whether to try and bootstrap all the discovered Zen2 nodes when the last Zen1 node leaves the cluster. - public static final Setting ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING = - Setting.boolSetting("discovery.zen.unsafe_rolling_upgrades_enabled", true, Setting.Property.NodeScope, Setting.Property.Deprecated); - - /** - * Dummy {@link ElectMasterService} that is only used to choose the best 6.x master from the discovered nodes, ignoring the - * `minimum_master_nodes` setting. - */ - private static final ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY); - - private final TransportService transportService; - private final BooleanSupplier isBootstrappedSupplier; - private final JoinHelper joinHelper; - private final Supplier> peersSupplier; - private final Consumer initialConfigurationConsumer; - private final TimeValue bwcPingTimeout; - private final boolean enableUnsafeBootstrappingOnUpgrade; - private final ClusterName clusterName; - - @Nullable // null if no active joining round - private volatile JoiningRound joiningRound; - - public DiscoveryUpgradeService(Settings settings, TransportService transportService, - BooleanSupplier isBootstrappedSupplier, JoinHelper joinHelper, - Supplier> peersSupplier, - Consumer initialConfigurationConsumer) { - assert Version.CURRENT.major == Version.V_6_6_0.major + 1 : "remove this service once unsafe upgrades are no longer needed"; - this.transportService = transportService; - this.isBootstrappedSupplier = isBootstrappedSupplier; - this.joinHelper = joinHelper; - this.peersSupplier = peersSupplier; - this.initialConfigurationConsumer = initialConfigurationConsumer; - this.bwcPingTimeout = BWC_PING_TIMEOUT_SETTING.get(settings); - this.enableUnsafeBootstrappingOnUpgrade = ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING.get(settings); - this.clusterName = CLUSTER_NAME_SETTING.get(settings); - } - - public void activate(Optional lastKnownLeader, ClusterState lastAcceptedClusterState) { - // called under coordinator mutex - - if (isBootstrappedSupplier.getAsBoolean()) { - return; - } - - assert lastKnownLeader.isPresent() == false || Coordinator.isZen1Node(lastKnownLeader.get()) : lastKnownLeader; - // if there was a leader and it's not a old node then we must have been bootstrapped - - final Settings dynamicSettings = lastAcceptedClusterState.metaData().settings(); - final int minimumMasterNodes = DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(dynamicSettings) - ? DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(dynamicSettings) - : lastAcceptedClusterState.getMinimumMasterNodesOnPublishingMaster(); - - assert joiningRound == null : joiningRound; - final Set knownMasterNodeIds = new HashSet<>(); - lastAcceptedClusterState.nodes().getMasterNodes().forEach(c -> knownMasterNodeIds.add(c.key)); - - joiningRound - = new JoiningRound(enableUnsafeBootstrappingOnUpgrade && lastKnownLeader.isPresent(), minimumMasterNodes, knownMasterNodeIds); - joiningRound.scheduleNextAttempt(); - } - - public void deactivate() { - // called under coordinator mutex - joiningRound = null; - } - - /** - * Waits for some number of calls to {@link ListenableCountDown#countDown()} and then notifies a listener. The listener - * is only ever notified once, whether successful or not. - */ - private static class ListenableCountDown { - private final CountDown countDown; - private final ActionListener listener; - - ListenableCountDown(int count, ActionListener listener) { - this.countDown = new CountDown(count); - this.listener = listener; - } - - void onFailure(Exception e) { - if (countDown.fastForward()) { - listener.onFailure(e); - } - } - - void countDown() { - if (countDown.countDown()) { - listener.onResponse(null); - } - } - } - - private class JoiningRound { - private final boolean upgrading; - private final int minimumMasterNodes; - private final Set knownMasterNodeIds; - - JoiningRound(boolean upgrading, int minimumMasterNodes, Set knownMasterNodeIds) { - this.upgrading = upgrading; - this.minimumMasterNodes = minimumMasterNodes; - this.knownMasterNodeIds = knownMasterNodeIds; - } - - private boolean isRunning() { - return joiningRound == this && isBootstrappedSupplier.getAsBoolean() == false; - } - - private boolean canBootstrap(Set discoveryNodes) { - return upgrading && minimumMasterNodes <= discoveryNodes.stream().filter(DiscoveryNode::isMasterNode).count(); - } - - void scheduleNextAttempt() { - if (isRunning() == false) { - return; - } - - final ThreadPool threadPool = transportService.getThreadPool(); - threadPool.scheduleUnlessShuttingDown(bwcPingTimeout, Names.SAME, new Runnable() { - - @Override - public void run() { - if (isRunning() == false) { - return; - } - - final Set discoveryNodes = Stream.concat(StreamSupport.stream(peersSupplier.get().spliterator(), false), - Stream.of(transportService.getLocalNode())).filter(DiscoveryNode::isMasterNode).collect(Collectors.toSet()); - - // this set of nodes is reasonably fresh - the PeerFinder cleans up nodes to which the transport service is not - // connected each time it wakes up (every second by default) - - logger.debug("upgrading={}, minimumMasterNodes={}, nodes={}", upgrading, minimumMasterNodes, discoveryNodes); - - if (discoveryNodes.stream().anyMatch(Coordinator::isZen1Node)) { - electBestOldMaster(discoveryNodes); - } else if (canBootstrap(discoveryNodes)) { - // no Zen1 nodes found, but the last-known master was a Zen1 node, so this is a rolling upgrade - transportService.getThreadPool().generic().execute(() -> { - try { - Set nodeIds = new HashSet<>(); - discoveryNodes.forEach(n -> nodeIds.add(n.getId())); - - final Iterator knownNodeIdIterator = knownMasterNodeIds.iterator(); - while (nodeIds.size() < 2 * minimumMasterNodes - 1 && knownNodeIdIterator.hasNext()) { - nodeIds.add(knownNodeIdIterator.next()); - } - - final VotingConfiguration votingConfiguration = new VotingConfiguration(nodeIds); - assert votingConfiguration.hasQuorum( - discoveryNodes.stream().map(DiscoveryNode::getId).collect(Collectors.toList())); - assert 2 * minimumMasterNodes - 2 <= nodeIds.size() : nodeIds + " too small for " + minimumMasterNodes; - - initialConfigurationConsumer.accept(votingConfiguration); - } catch (Exception e) { - logger.debug("exception during bootstrapping upgrade, retrying", e); - } finally { - scheduleNextAttempt(); - } - }); - } else { - scheduleNextAttempt(); - } - } - - /** - * Ping all the old discovered masters one more time to obtain their cluster state versions, and then vote for the best one. - * @param discoveryNodes The master nodes (old and new). - */ - private void electBestOldMaster(Set discoveryNodes) { - final Set masterCandidates = newConcurrentSet(); - final ListenableCountDown listenableCountDown - = new ListenableCountDown(discoveryNodes.size(), new ActionListener() { - - @Override - public void onResponse(Void value) { - assert masterCandidates.size() == discoveryNodes.size() - : masterCandidates + " does not match " + discoveryNodes; - - // TODO we shouldn't elect a master with a version that's older than ours - // If the only Zen1 nodes left are stale, and we can bootstrap, maybe we should bootstrap? - // Do we ever need to elect a freshly-started Zen1 node? - if (isRunning()) { - final MasterCandidate electedMaster = electMasterService.electMaster(masterCandidates); - logger.debug("elected {}, sending join", electedMaster); - joinHelper.sendJoinRequest(electedMaster.getNode(), Optional.empty(), - JoiningRound.this::scheduleNextAttempt); - } - } - - @Override - public void onFailure(Exception e) { - scheduleNextAttempt(); - } - }); - - boolean foundOldMaster = false; - for (final DiscoveryNode discoveryNode : discoveryNodes) { - assert discoveryNode.isMasterNode() : discoveryNode; - if (Coordinator.isZen1Node(discoveryNode)) { - foundOldMaster = true; - transportService.sendRequest(discoveryNode, UnicastZenPing.ACTION_NAME, - new UnicastPingRequest(0, TimeValue.ZERO, - new PingResponse(createDiscoveryNodeWithImpossiblyHighId(transportService.getLocalNode()), - null, clusterName, UNKNOWN_VERSION)), - TransportRequestOptions.builder().withTimeout(bwcPingTimeout).build(), - new TransportResponseHandler() { - @Override - public void handleResponse(UnicastPingResponse response) { - long clusterStateVersion = UNKNOWN_VERSION; - for (PingResponse pingResponse : response.pingResponses) { - if (discoveryNode.equals(pingResponse.node())) { - clusterStateVersion - = max(clusterStateVersion, pingResponse.getClusterStateVersion()); - } - } - masterCandidates.add(new MasterCandidate(discoveryNode, clusterStateVersion)); - listenableCountDown.countDown(); - } - - @Override - public void handleException(TransportException exp) { - logger.debug( - new ParameterizedMessage("unexpected exception when pinging {}", discoveryNode), exp); - listenableCountDown.onFailure(exp); - } - - @Override - public String executor() { - return Names.SAME; - } - - @Override - public UnicastPingResponse read(StreamInput in) throws IOException { - return new UnicastPingResponse(in); - } - }); - - } else { - masterCandidates.add( - new MasterCandidate(createDiscoveryNodeWithImpossiblyHighId(discoveryNode), UNKNOWN_VERSION)); - listenableCountDown.countDown(); - } - } - assert foundOldMaster; - } - - @Override - public String toString() { - return "discovery upgrade service retry"; - } - }); - } - } - - /** - * Pre-7.0 nodes select the best master by comparing their IDs (as strings) and selecting the lowest one amongst those nodes with - * the best cluster state version. We want 7.0+ nodes to participate in these elections in a mixed cluster but never to win one, so - * we lie and claim to have an impossible ID that compares above all genuine IDs. - */ - public static DiscoveryNode createDiscoveryNodeWithImpossiblyHighId(DiscoveryNode node) { - // IDs are base-64-encoded UUIDs, which means they use the character set [0-9A-Za-z_-]. The highest character in this set is 'z', - // and 'z' < '{', so by starting the ID with '{' we can be sure it's greater. This is terrible. - return new DiscoveryNode(node.getName(), "{zen2}" + node.getId(), node.getEphemeralId(), node.getHostName(), - node.getHostAddress(), node.getAddress(), node.getAttributes(), node.getRoles(), node.getVersion()); - } -} diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java index b0b91cd0980f2..241b9b3f82231 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java @@ -93,31 +93,9 @@ public PublicationTransportHandler(TransportService transportService, NamedWrite transportService.registerRequestHandler(PUBLISH_STATE_ACTION_NAME, BytesTransportRequest::new, ThreadPool.Names.GENERIC, false, false, (request, channel, task) -> channel.sendResponse(handleIncomingPublishRequest(request))); - transportService.registerRequestHandler(PublishClusterStateAction.SEND_ACTION_NAME, BytesTransportRequest::new, - ThreadPool.Names.GENERIC, - false, false, (request, channel, task) -> { - handleIncomingPublishRequest(request); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); - transportService.registerRequestHandler(COMMIT_STATE_ACTION_NAME, ThreadPool.Names.GENERIC, false, false, ApplyCommitRequest::new, (request, channel, task) -> handleApplyCommit.accept(request, transportCommitCallback(channel))); - - transportService.registerRequestHandler(PublishClusterStateAction.COMMIT_ACTION_NAME, - PublishClusterStateAction.CommitClusterStateRequest::new, - ThreadPool.Names.GENERIC, false, false, - (request, channel, task) -> { - final Optional matchingClusterState = Optional.ofNullable(lastSeenClusterState.get()).filter( - cs -> cs.stateUUID().equals(request.stateUUID)); - if (matchingClusterState.isPresent() == false) { - throw new IllegalStateException("can't resolve cluster state with uuid" + - " [" + request.stateUUID + "] to commit"); - } - final ApplyCommitRequest applyCommitRequest = new ApplyCommitRequest(matchingClusterState.get().getNodes().getMasterNode(), - matchingClusterState.get().term(), matchingClusterState.get().version()); - handleApplyCommit.accept(applyCommitRequest, transportCommitCallback(channel)); - }); } private ActionListener transportCommitCallback(TransportChannel channel) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 0ae30c465b26e..61c3dd9adadad 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -35,7 +35,6 @@ import org.elasticsearch.cluster.coordination.ClusterBootstrapService; import org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper; import org.elasticsearch.cluster.coordination.Coordinator; -import org.elasticsearch.cluster.coordination.DiscoveryUpgradeService; import org.elasticsearch.cluster.coordination.ElectionSchedulerFactory; import org.elasticsearch.cluster.coordination.FollowersChecker; import org.elasticsearch.cluster.coordination.JoinHelper; @@ -67,9 +66,9 @@ import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.PeerFinder; +import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider; import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.FaultDetection; -import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider; import org.elasticsearch.discovery.zen.UnicastZenPing; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.env.Environment; @@ -493,9 +492,7 @@ public void apply(Settings value, Settings current, Settings previous) { TransportAddVotingConfigExclusionsAction.MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING, ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING, ClusterBootstrapService.UNCONFIGURED_BOOTSTRAP_TIMEOUT_SETTING, - LagDetector.CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING, - DiscoveryUpgradeService.BWC_PING_TIMEOUT_SETTING, - DiscoveryUpgradeService.ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING + LagDetector.CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING ))); public static List> BUILT_IN_SETTING_UPGRADERS = Collections.unmodifiableList(Arrays.asList( diff --git a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java index 58248b8183d34..839661ca87e8a 100644 --- a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java @@ -25,8 +25,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.coordination.PeersResponse; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -37,24 +35,13 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.discovery.zen.UnicastZenPing; -import org.elasticsearch.discovery.zen.ZenDiscovery; -import org.elasticsearch.discovery.zen.ZenPing; -import org.elasticsearch.discovery.zen.ZenPing.PingResponse; -import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool.Names; -import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -64,8 +51,6 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; -import static org.elasticsearch.cluster.coordination.Coordinator.isZen1Node; -import static org.elasticsearch.cluster.coordination.DiscoveryUpgradeService.createDiscoveryNodeWithImpossiblyHighId; public abstract class PeerFinder { @@ -82,8 +67,6 @@ public abstract class PeerFinder { Setting.timeSetting("discovery.request_peers_timeout", TimeValue.timeValueMillis(3000), TimeValue.timeValueMillis(1), Setting.Property.NodeScope); - private final Settings settings; - private final TimeValue findPeersInterval; private final TimeValue requestPeersTimeout; @@ -101,7 +84,6 @@ public abstract class PeerFinder { public PeerFinder(Settings settings, TransportService transportService, TransportAddressConnector transportAddressConnector, ConfiguredHostsResolver configuredHostsResolver) { - this.settings = settings; findPeersInterval = DISCOVERY_FIND_PEERS_INTERVAL_SETTING.get(settings); requestPeersTimeout = DISCOVERY_REQUEST_PEERS_TIMEOUT_SETTING.get(settings); this.transportService = transportService; @@ -111,9 +93,6 @@ public PeerFinder(Settings settings, TransportService transportService, Transpor transportService.registerRequestHandler(REQUEST_PEERS_ACTION_NAME, Names.GENERIC, false, false, PeersRequest::new, (request, channel, task) -> channel.sendResponse(handlePeersRequest(request))); - - transportService.registerRequestHandler(UnicastZenPing.ACTION_NAME, Names.GENERIC, false, false, - UnicastZenPing.UnicastPingRequest::new, new Zen1UnicastPingRequestHandler()); } public void activate(final DiscoveryNodes lastAcceptedNodes) { @@ -453,36 +432,10 @@ public String executor() { return Names.GENERIC; } }; - final String actionName; - final TransportRequest transportRequest; - final TransportResponseHandler transportResponseHandler; - if (isZen1Node(discoveryNode)) { - actionName = UnicastZenPing.ACTION_NAME; - transportRequest = new UnicastZenPing.UnicastPingRequest(1, ZenDiscovery.PING_TIMEOUT_SETTING.get(settings), - new ZenPing.PingResponse(createDiscoveryNodeWithImpossiblyHighId(getLocalNode()), null, - ClusterName.CLUSTER_NAME_SETTING.get(settings), ClusterState.UNKNOWN_VERSION)); - transportResponseHandler = peersResponseHandler.wrap(ucResponse -> { - Optional optionalMasterNode = Arrays.stream(ucResponse.pingResponses) - .filter(pr -> discoveryNode.equals(pr.node()) && discoveryNode.equals(pr.master())) - .map(ZenPing.PingResponse::node) - .findFirst(); - List discoveredNodes = new ArrayList<>(); - if (optionalMasterNode.isPresent() == false) { - Arrays.stream(ucResponse.pingResponses).map(PingResponse::master).filter(Objects::nonNull) - .forEach(discoveredNodes::add); - Arrays.stream(ucResponse.pingResponses).map(PingResponse::node).forEach(discoveredNodes::add); - } - return new PeersResponse(optionalMasterNode, discoveredNodes, 0L); - }, UnicastZenPing.UnicastPingResponse::new); - } else { - actionName = REQUEST_PEERS_ACTION_NAME; - transportRequest = new PeersRequest(getLocalNode(), knownNodes); - transportResponseHandler = peersResponseHandler; - } - transportService.sendRequest(discoveryNode, actionName, - transportRequest, + transportService.sendRequest(discoveryNode, REQUEST_PEERS_ACTION_NAME, + new PeersRequest(getLocalNode(), knownNodes), TransportRequestOptions.builder().withTimeout(requestPeersTimeout).build(), - transportResponseHandler); + peersResponseHandler); } @Override @@ -494,22 +447,4 @@ public String toString() { '}'; } } - - private class Zen1UnicastPingRequestHandler implements TransportRequestHandler { - @Override - public void messageReceived(UnicastZenPing.UnicastPingRequest request, TransportChannel channel, Task task) throws Exception { - final PeersRequest peersRequest = new PeersRequest(request.pingResponse.node(), - Optional.ofNullable(request.pingResponse.master()).map(Collections::singletonList).orElse(emptyList())); - final PeersResponse peersResponse = handlePeersRequest(peersRequest); - final List pingResponses = new ArrayList<>(); - final ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings); - pingResponses.add(new ZenPing.PingResponse(createDiscoveryNodeWithImpossiblyHighId(transportService.getLocalNode()), - peersResponse.getMasterNode().orElse(null), - clusterName, ClusterState.UNKNOWN_VERSION)); - peersResponse.getKnownPeers().forEach(dn -> pingResponses.add( - new ZenPing.PingResponse(ZenPing.PingResponse.FAKE_PING_ID, - isZen1Node(dn) ? dn : createDiscoveryNodeWithImpossiblyHighId(dn), null, clusterName, ClusterState.UNKNOWN_VERSION))); - channel.sendResponse(new UnicastZenPing.UnicastPingResponse(request.id, pingResponses.toArray(new ZenPing.PingResponse[0]))); - } - } } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeServiceTests.java deleted file mode 100644 index 01e9c1403313c..0000000000000 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeServiceTests.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.cluster.coordination; - -import org.elasticsearch.Version; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.test.ESTestCase; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.lessThan; - -public class DiscoveryUpgradeServiceTests extends ESTestCase { - public void testCreateDiscoveryNodeWithImpossiblyHighId() { - final DiscoveryNode discoveryNode - = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT); - final DiscoveryNode fakeNode = DiscoveryUpgradeService.createDiscoveryNodeWithImpossiblyHighId(discoveryNode); - assertThat(discoveryNode.getId(), lessThan(fakeNode.getId())); - assertThat(UUIDs.randomBase64UUID(random()), lessThan(fakeNode.getId())); - assertThat(fakeNode.getId(), containsString(discoveryNode.getId())); - } -} diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java deleted file mode 100644 index 3ba9f58523d16..0000000000000 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.cluster.coordination; - -import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; -import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; -import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction; -import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.Requests; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.Manifest; -import org.elasticsearch.cluster.routing.UnassignedInfo; -import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.Allocation; -import org.elasticsearch.common.Priority; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.Discovery; -import org.elasticsearch.discovery.DiscoveryModule; -import org.elasticsearch.discovery.zen.ElectMasterService; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.gateway.MetaStateService; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster.RestartCallback; -import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.TransportService; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.StreamSupport; - -import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; -import static org.elasticsearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM; -import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_ACTION_NAME; -import static org.elasticsearch.cluster.coordination.JoinHelper.START_JOIN_ACTION_NAME; -import static org.elasticsearch.cluster.coordination.PublicationTransportHandler.PUBLISH_STATE_ACTION_NAME; -import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING; -import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING; -import static org.elasticsearch.node.Node.NODE_NAME_SETTING; -import static org.elasticsearch.test.InternalTestCluster.REMOVED_MINIMUM_MASTER_NODES; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) -public class Zen1IT extends ESIntegTestCase { - - private static Settings ZEN1_SETTINGS = Coordinator.addZen1Attribute(true, Settings.builder() - .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.ZEN_DISCOVERY_TYPE)).build(); - - private static Settings ZEN2_SETTINGS = Settings.builder() - .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.ZEN2_DISCOVERY_TYPE) - .build(); - - protected Collection> nodePlugins() { - return Collections.singletonList(MockTransportService.TestPlugin.class); - } - - public void testZen2NodesJoiningZen1Cluster() { - internalCluster().startNodes(randomIntBetween(1, 3), ZEN1_SETTINGS); - internalCluster().startNodes(randomIntBetween(1, 3), ZEN2_SETTINGS); - createIndex("test"); - } - - public void testZen1NodesJoiningZen2Cluster() { - internalCluster().startNodes(randomIntBetween(1, 3), ZEN2_SETTINGS); - internalCluster().startNodes(randomIntBetween(1, 3), ZEN1_SETTINGS); - createIndex("test"); - } - - public void testMixedClusterDisruption() throws Exception { - final List nodes = internalCluster().startNodes(IntStream.range(0, 5) - .mapToObj(i -> i < 2 ? ZEN1_SETTINGS : ZEN2_SETTINGS).toArray(Settings[]::new)); - - final List transportServices = nodes.stream() - .map(n -> (MockTransportService) internalCluster().getInstance(TransportService.class, n)).collect(Collectors.toList()); - - logger.info("--> disrupting communications"); - - // The idea here is to make some of the Zen2 nodes believe the Zen1 nodes have gone away by introducing a network partition, so that - // they bootstrap themselves, but keep the Zen1 side of the cluster alive. - - // Set up a bridged network partition with the Zen1 nodes {0,1} on one side, Zen2 nodes {3,4} on the other, and node {2} in both - transportServices.get(0).addFailToSendNoConnectRule(transportServices.get(3)); - transportServices.get(0).addFailToSendNoConnectRule(transportServices.get(4)); - transportServices.get(1).addFailToSendNoConnectRule(transportServices.get(3)); - transportServices.get(1).addFailToSendNoConnectRule(transportServices.get(4)); - transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(0)); - transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(1)); - transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(0)); - transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(1)); - - // Nodes 3 and 4 will bootstrap, but we want to keep node 2 as part of the Zen1 cluster, so prevent any messages that might switch - // its allegiance - transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(2), - PUBLISH_STATE_ACTION_NAME, FOLLOWER_CHECK_ACTION_NAME, START_JOIN_ACTION_NAME); - transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(2), - PUBLISH_STATE_ACTION_NAME, FOLLOWER_CHECK_ACTION_NAME, START_JOIN_ACTION_NAME); - - logger.info("--> waiting for disconnected nodes to be removed"); - ensureStableCluster(3, nodes.get(0)); - - logger.info("--> creating index on Zen1 side"); - assertAcked(client(nodes.get(0)).admin().indices().create(new CreateIndexRequest("test")).get()); - assertFalse(client(nodes.get(0)).admin().cluster().health(new ClusterHealthRequest("test") - .waitForGreenStatus()).get().isTimedOut()); - - logger.info("--> waiting for disconnected nodes to bootstrap themselves"); - assertBusy(() -> assertTrue(IntStream.range(3, 5) - .mapToObj(n -> (Coordinator) internalCluster().getInstance(Discovery.class, nodes.get(n))) - .anyMatch(Coordinator::isInitialConfigurationSet))); - - logger.info("--> clearing disruption and waiting for cluster to reform"); - transportServices.forEach(MockTransportService::clearAllRules); - - ensureStableCluster(5, nodes.get(0)); - assertFalse(client(nodes.get(0)).admin().cluster().health(new ClusterHealthRequest("test") - .waitForGreenStatus()).get().isTimedOut()); - } - - public void testMixedClusterFormation() throws Exception { - final int zen1NodeCount = randomIntBetween(1, 3); - final int zen2NodeCount = randomIntBetween(zen1NodeCount == 1 ? 2 : 1, 3); - logger.info("starting cluster of [{}] Zen1 nodes and [{}] Zen2 nodes", zen1NodeCount, zen2NodeCount); - final List nodes = internalCluster().startNodes(IntStream.range(0, zen1NodeCount + zen2NodeCount) - .mapToObj(i -> i < zen1NodeCount ? ZEN1_SETTINGS : ZEN2_SETTINGS).toArray(Settings[]::new)); - - createIndex("test", - Settings.builder() - .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) // assign shards - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, zen1NodeCount + zen2NodeCount + randomIntBetween(0, 2)) // causes rebalancing - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .build()); - ensureGreen("test"); - - for (final String node : nodes) { - // With 1 Zen1 node when you stop the Zen1 node the Zen2 nodes might auto-bootstrap. - // But there are only 2 Zen2 nodes so you must do the right things with voting config exclusions to keep the cluster - // alive through the other two restarts. - final boolean masterNodeIsZen2 = zen1NodeCount <= nodes.indexOf(internalCluster().getMasterName()); - final boolean thisNodeIsZen2 = zen1NodeCount <= nodes.indexOf(node); - final boolean requiresVotingConfigExclusions = zen1NodeCount == 1 && zen2NodeCount == 2 && masterNodeIsZen2 && thisNodeIsZen2; - - if (requiresVotingConfigExclusions) { - client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(new String[]{node})).get(); - } - - internalCluster().restartNode(node, new RestartCallback() { - @Override - public Settings onNodeStopped(String restartingNode) { - String viaNode = randomValueOtherThan(restartingNode, () -> randomFrom(nodes)); - final ClusterHealthRequestBuilder clusterHealthRequestBuilder = client(viaNode).admin().cluster().prepareHealth() - .setWaitForEvents(Priority.LANGUID) - .setWaitForNodes(Integer.toString(zen1NodeCount + zen2NodeCount - 1)) - .setTimeout(TimeValue.timeValueSeconds(30)); - ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get(); - assertFalse(restartingNode, clusterHealthResponse.isTimedOut()); - return Settings.EMPTY; - } - }); - ensureStableCluster(zen1NodeCount + zen2NodeCount); - ensureGreen("test"); - - if (requiresVotingConfigExclusions) { - final ClearVotingConfigExclusionsRequest clearVotingTombstonesRequest = new ClearVotingConfigExclusionsRequest(); - clearVotingTombstonesRequest.setWaitForRemoval(false); - client().execute(ClearVotingConfigExclusionsAction.INSTANCE, clearVotingTombstonesRequest).get(); - } - } - } - - public void testRollingMigrationFromZen1ToZen2() throws Exception { - final int nodeCount = randomIntBetween(2, 5); - final List zen1Nodes = internalCluster().startNodes(nodeCount, ZEN1_SETTINGS); - - createIndex("test", - Settings.builder() - .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) // assign shards - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, nodeCount) // causes rebalancing - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .build()); - ensureGreen("test"); - - for (final String zen1Node : zen1Nodes) { - logger.info("--> shutting down {}", zen1Node); - internalCluster().stopRandomNode(s -> NODE_NAME_SETTING.get(s).equals(zen1Node)); - - ensureStableCluster(nodeCount - 1); - if (nodeCount > 2) { - ensureGreen("test"); - } else { - ensureYellow("test"); - } - - logger.info("--> starting replacement for {}", zen1Node); - final String newNode = internalCluster().startNode(ZEN2_SETTINGS); - ensureStableCluster(nodeCount); - ensureGreen("test"); - logger.info("--> successfully replaced {} with {}", zen1Node, newNode); - } - - assertThat(internalCluster().size(), equalTo(nodeCount)); - } - - public void testRollingUpgradeFromZen1ToZen2() throws Exception { - final int nodeCount = randomIntBetween(2, 5); - final List nodes = internalCluster().startNodes(nodeCount, ZEN1_SETTINGS); - - createIndex("test", - Settings.builder() - .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) // assign shards - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, nodeCount) // causes rebalancing - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .build()); - ensureGreen("test"); - - internalCluster().rollingRestart(new RestartCallback() { - @Override - public void doAfterNodes(int n, Client client) { - ensureGreen("test"); - } - - @Override - public Settings onNodeStopped(String nodeName) { - String viaNode = randomValueOtherThan(nodeName, () -> randomFrom(nodes)); - final ClusterHealthRequestBuilder clusterHealthRequestBuilder = client(viaNode).admin().cluster().prepareHealth() - .setWaitForEvents(Priority.LANGUID) - .setWaitForNodes(Integer.toString(nodeCount - 1)) - .setTimeout(TimeValue.timeValueSeconds(30)); - if (nodeCount == 2) { - clusterHealthRequestBuilder.setWaitForYellowStatus(); - } else { - clusterHealthRequestBuilder.setWaitForGreenStatus(); - } - ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get(); - assertFalse(nodeName, clusterHealthResponse.isTimedOut()); - return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES)).build(); - } - }); - - ensureStableCluster(nodeCount); - ensureGreen("test"); - assertThat(internalCluster().size(), equalTo(nodeCount)); - } - - private void testMultipleNodeMigrationFromZen1ToZen2(int nodeCount) throws Exception { - final List oldNodes = internalCluster().startNodes(nodeCount, ZEN1_SETTINGS); - createIndex("test", - Settings.builder() - .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) // assign shards - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, nodeCount) // causes rebalancing - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, nodeCount > 1 ? 1 : 0) - .build()); - ensureGreen("test"); - - internalCluster().startNodes(nodeCount, ZEN2_SETTINGS); - - logger.info("--> updating settings to exclude old nodes"); - client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder() - .put(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), String.join(",", oldNodes))).get(); - - logger.info("--> waiting for old nodes to be vacated"); - waitForRelocation(); - - while (internalCluster().size() > nodeCount) { - internalCluster().stopRandomNode(settings -> oldNodes.contains(NODE_NAME_SETTING.get(settings))); - } - - ensureGreen("test"); - } - - public void testMultipleNodeMigrationFromZen1ToZen2WithOneNode() throws Exception { - testMultipleNodeMigrationFromZen1ToZen2(1); - } - - public void testMultipleNodeMigrationFromZen1ToZen2WithTwoNodes() throws Exception { - testMultipleNodeMigrationFromZen1ToZen2(2); - } - - public void testMultipleNodeMigrationFromZen1ToZen2WithThreeNodes() throws Exception { - testMultipleNodeMigrationFromZen1ToZen2(3); - } - - public void testFreshestMasterElectedAfterFullClusterRestart() throws Exception { - final List nodeNames = internalCluster().startNodes(3, ZEN1_SETTINGS); - - assertTrue(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder() - .put(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), Allocation.ALL)).get().isAcknowledged()); - - final List nodeEnvironments - = StreamSupport.stream(internalCluster().getDataOrMasterNodeInstances(NodeEnvironment.class).spliterator(), false) - .collect(Collectors.toList()); - - final boolean randomiseVersions = rarely(); - - internalCluster().fullRestart(new RestartCallback() { - int nodesStopped; - - @Override - public Settings onNodeStopped(String nodeName) throws Exception { - nodesStopped += 1; - - if (nodesStopped == 1) { - final Client client = internalCluster().client(randomValueOtherThan(nodeName, () -> randomFrom(nodeNames))); - - assertFalse(client.admin().cluster().health(Requests.clusterHealthRequest() - .waitForEvents(Priority.LANGUID) - .waitForNoRelocatingShards(true) - .waitForNodes("2")).actionGet().isTimedOut()); - - assertTrue(client.admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder() - .put(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), Allocation.NONE)).get().isAcknowledged()); - } - - if (nodesStopped == nodeNames.size()) { - for (final NodeEnvironment nodeEnvironment : nodeEnvironments) { - // The versions written by nodes following a Zen1 master cannot be trusted. Randomise them to demonstrate they are - // not important. - final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry()); - final Manifest manifest = metaStateService.loadManifestOrEmpty(); - assertThat(manifest.getCurrentTerm(), is(ZEN1_BWC_TERM)); - final long newVersion = randomiseVersions ? randomNonNegativeLong() : 0L; - metaStateService.writeManifestAndCleanup("altering version to " + newVersion, - new Manifest(manifest.getCurrentTerm(), newVersion, manifest.getGlobalGeneration(), - manifest.getIndexGenerations())); - } - } - - return Coordinator.addZen1Attribute(false, Settings.builder()) - .put(ZEN2_SETTINGS) - .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES) - .build(); - } - }); - - assertFalse(client().admin().cluster().health(Requests.clusterHealthRequest() - .waitForEvents(Priority.LANGUID) - .waitForNoRelocatingShards(true) - .waitForNodes("3")).actionGet().isTimedOut()); - - assertThat(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.get( - client().admin().cluster().state(new ClusterStateRequest()).get().getState().metaData().settings()), - equalTo(Allocation.NONE)); - } -} From ced375e25f5a259ab4e0131a2775e608082fc2a8 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 17:34:00 -0500 Subject: [PATCH 07/43] Remove version method --- .../node/InternalSettingsPreparer.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java index 623b763e84989..ca7004fe03dc1 100644 --- a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java +++ b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java @@ -88,7 +88,6 @@ public static Environment prepareEnvironment(Settings input, Map // re-initialize settings now that the config file has been loaded initializeSettings(output, input, properties); - checkSettingsForTerminalDeprecation(output); finalizeSettings(output, defaultNodeName); environment = new Environment(output.build(), configPath); @@ -112,25 +111,6 @@ static void initializeSettings(final Settings.Builder output, final Settings inp output.replacePropertyPlaceholders(); } - /** - * Checks all settings values to make sure they do not have the old prompt settings. These were deprecated in 6.0.0. - * This check should be removed in 8.0.0. - */ - private static void checkSettingsForTerminalDeprecation(final Settings.Builder output) throws SettingsException { - // This method to be removed in 8.0.0, as it was deprecated in 6.0 and removed in 7.0 - // assert Version.CURRENT.major != 8: "Logic pertaining to config driven prompting should be removed"; - for (String setting : output.keys()) { - switch (output.get(setting)) { - case SECRET_PROMPT_VALUE: - throw new SettingsException("Config driven secret prompting was deprecated in 6.0.0. Use the keystore" + - " for secure settings."); - case TEXT_PROMPT_VALUE: - throw new SettingsException("Config driven text prompting was deprecated in 6.0.0. Use the keystore" + - " for secure settings."); - } - } - } - /** * Finish preparing settings by replacing forced settings and any defaults that need to be added. */ From 6d84429a497aabd4dd465b038e9fed2801840321 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 17:36:25 -0500 Subject: [PATCH 08/43] Remove unused fields --- .../java/org/elasticsearch/node/InternalSettingsPreparer.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java index ca7004fe03dc1..e2c33e8cfec94 100644 --- a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java +++ b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java @@ -38,9 +38,6 @@ public class InternalSettingsPreparer { - private static final String SECRET_PROMPT_VALUE = "${prompt.secret}"; - private static final String TEXT_PROMPT_VALUE = "${prompt.text}"; - /** * Prepares settings for the transport client by gathering all * elasticsearch system properties and setting defaults. From d7748f47ab3386e5bab11f758c39199416c0ea7c Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 17:37:00 -0500 Subject: [PATCH 09/43] Remove unused import --- .../src/main/java/org/elasticsearch/test/VersionUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index dbd786996bb15..f775f9f5b0121 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.collect.Tuple; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; From f12c54a890cc7deecbeaa108af328aefb14a75c1 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 17:38:30 -0500 Subject: [PATCH 10/43] Remove more unused imports --- .../node/InternalSettingsPreparer.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java index e2c33e8cfec94..b73268d9985b5 100644 --- a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java +++ b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java @@ -19,6 +19,11 @@ package org.elasticsearch.node; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsException; +import org.elasticsearch.env.Environment; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -26,15 +31,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; import java.util.function.Function; - -import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsException; -import org.elasticsearch.env.Environment; -import org.elasticsearch.node.Node; +import java.util.function.Supplier; public class InternalSettingsPreparer { From 64f11fc7deabf4df5219e572398910877adacfa5 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 18:00:24 -0500 Subject: [PATCH 11/43] Remove bogus assertion --- server/src/test/java/org/elasticsearch/VersionTests.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/VersionTests.java b/server/src/test/java/org/elasticsearch/VersionTests.java index 08db8dfaf2100..99077cbea70db 100644 --- a/server/src/test/java/org/elasticsearch/VersionTests.java +++ b/server/src/test/java/org/elasticsearch/VersionTests.java @@ -367,10 +367,9 @@ public void testIsCompatible() { () -> new IllegalStateException("expected previous minor version for [" + currentOrNextMajorVersion + "]")); final Version previousMinorVersion = VersionUtils.getPreviousMinorVersion(); - assert previousMinorVersion.major == currentOrNextMajorVersion.major - || previousMinorVersion.major == lastMinorFromPreviousMajor.major; - boolean isCompatible = previousMinorVersion.major == currentOrNextMajorVersion.major - || previousMinorVersion.minor == lastMinorFromPreviousMajor.minor; + boolean isCompatible = + previousMinorVersion.major == currentOrNextMajorVersion.major + || previousMinorVersion.minor == lastMinorFromPreviousMajor.minor; final String message = String.format( Locale.ROOT, From a54cf6cd10e19d077e32df5ad234d8c30d11587f Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 18:24:45 -0500 Subject: [PATCH 12/43] Use typeless APIs --- .../upgrades/FullClusterRestartIT.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 9740ff4222d7e..070620b8e59f0 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -510,7 +510,7 @@ public void testRollover() throws IOException { bulk.append("{\"index\":{}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - Request bulkRequest = new Request("POST", "/" + index + "_write/doc/_bulk"); + Request bulkRequest = new Request("POST", "/" + index + "_write/_doc/_bulk"); bulkRequest.setJsonEntity(bulk.toString()); bulkRequest.addParameter("refresh", ""); bulkRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); @@ -644,12 +644,12 @@ void assertRealtimeGetWorks() throws IOException { Map hit = (Map) ((List)(XContentMapValues.extractValue("hits.hits", searchResponse))).get(0); String docId = (String) hit.get("_id"); - Request updateRequest = new Request("POST", "/" + index + "/doc/" + docId + "/_update"); + Request updateRequest = new Request("POST", "/" + index + "/_doc/" + docId + "/_update"); updateRequest.setOptions(expectWarnings(RestUpdateAction.TYPES_DEPRECATION_MESSAGE)); updateRequest.setJsonEntity("{ \"doc\" : { \"foo\": \"bar\"}}"); client().performRequest(updateRequest); - Request getRequest = new Request("GET", "/" + index + "/doc/" + docId); + Request getRequest = new Request("GET", "/" + index + "/_doc/" + docId); getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Map getRsp = entityAsMap(client().performRequest(getRequest)); Map source = (Map) getRsp.get("_source"); @@ -704,7 +704,7 @@ int extractTotalHits(Map response) { * Tests that a single document survives. Super basic smoke test. */ public void testSingleDoc() throws IOException { - String docLocation = "/" + index + "/doc/1"; + String docLocation = "/" + index + "/_doc/1"; String doc = "{\"test\": \"test\"}"; if (isRunningAgainstOldCluster()) { @@ -1016,7 +1016,7 @@ public void testSoftDeletes() throws Exception { int numDocs = between(10, 100); for (int i = 0; i < numDocs; i++) { String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject()); - Request request = new Request("POST", "/" + index + "/doc/" + i); + Request request = new Request("POST", "/" + index + "/_doc/" + i); request.setJsonEntity(doc); client().performRequest(request); if (rarely()) { @@ -1029,11 +1029,11 @@ public void testSoftDeletes() throws Exception { for (int i = 0; i < numDocs; i++) { if (randomBoolean()) { String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v2").endObject()); - Request request = new Request("POST", "/" + index + "/doc/" + i); + Request request = new Request("POST", "/" + index + "/_doc/" + i); request.setJsonEntity(doc); client().performRequest(request); } else if (randomBoolean()) { - client().performRequest(new Request("DELETE", "/" + index + "/doc/" + i)); + client().performRequest(new Request("DELETE", "/" + index + "/_doc/" + i)); liveDocs--; } } @@ -1103,7 +1103,7 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b bulk.append("{\"index\":{\"_id\":\"").append(count + i).append("\"}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/doc/_bulk"); + Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/_doc/_bulk"); writeToRestoredRequest.addParameter("refresh", "true"); writeToRestoredRequest.setJsonEntity(bulk.toString()); writeToRestoredRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); @@ -1166,7 +1166,7 @@ private void indexRandomDocuments(int count, boolean flushAllowed, boolean saveI logger.info("Indexing {} random documents", count); for (int i = 0; i < count; i++) { logger.debug("Indexing document [{}]", i); - Request createDocument = new Request("POST", "/" + index + "/doc/" + i); + Request createDocument = new Request("POST", "/" + index + "/_doc/" + i); createDocument.setJsonEntity(Strings.toString(docSupplier.apply(i))); client().performRequest(createDocument); if (rarely()) { @@ -1191,14 +1191,14 @@ private void saveInfoDocument(String type, String value) throws IOException { infoDoc.field("value", value); infoDoc.endObject(); // Only create the first version so we know how many documents are created when the index is first created - Request request = new Request("PUT", "/info/doc/" + index + "_" + type); + Request request = new Request("PUT", "/info/_doc/" + index + "_" + type); request.addParameter("op_type", "create"); request.setJsonEntity(Strings.toString(infoDoc)); client().performRequest(request); } private String loadInfoDocument(String type) throws IOException { - Request request = new Request("GET", "/info/doc/" + index + "_" + type); + Request request = new Request("GET", "/info/_doc/" + index + "_" + type); request.addParameter("filter_path", "_source"); request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); String doc = toStr(client().performRequest(request)); From a15a0418b4cd4fca25c9b7626187a06a9b70c6ad Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Thu, 7 Feb 2019 09:09:20 +0200 Subject: [PATCH 13/43] Remove copy_settings flag from resize operations This commit removes the `copy_settings` flag for shrink and split operations. The flag was introduced in https://github.com/elastic/elasticsearch/pull/30255 and the default value from now onwards is for the settings to be copied. --- .../upgrades/FullClusterRestartIT.java | 3 - .../rest-api-spec/api/indices.shrink.json | 4 - .../rest-api-spec/api/indices.split.json | 4 - .../test/indices.shrink/30_copy_settings.yml | 4 - .../test/indices.split/30_copy_settings.yml | 4 - .../admin/indices/RestResizeHandler.java | 25 ------ .../admin/indices/RestResizeHandlerTests.java | 77 ------------------- 7 files changed, 121 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandlerTests.java diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 9740ff4222d7e..cf8e86dda71d5 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -381,9 +381,6 @@ public void testShrink() throws IOException { client().performRequest(updateSettingsRequest); Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex); - if (getOldClusterVersion().onOrAfter(Version.V_6_4_0)) { - shrinkIndexRequest.addParameter("copy_settings", "true"); - } shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}"); client().performRequest(shrinkIndexRequest); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json index f92421b79ae91..5ef943eacba6c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json @@ -18,10 +18,6 @@ } }, "params": { - "copy_settings": { - "type" : "boolean", - "description" : "whether or not to copy settings from the source index (defaults to false)" - }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json index 2c14fced28c36..a79fa7b708269 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json @@ -18,10 +18,6 @@ } }, "params": { - "copy_settings": { - "type" : "boolean", - "description" : "whether or not to copy settings from the source index (defaults to false)" - }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml index 8c4c84c4be152..4cf77fe85caf0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml @@ -43,13 +43,10 @@ target: "copy-settings-target" wait_for_active_shards: 1 master_timeout: 10s - copy_settings: true body: settings: index.number_of_replicas: 0 index.merge.scheduler.max_thread_count: 2 - warnings: - - "parameter [copy_settings] is deprecated and will be removed in 8.0.0" - do: cluster.health: @@ -99,7 +96,6 @@ target: "explicit-no-copy-settings-target" wait_for_active_shards: 1 master_timeout: 10s - copy_settings: false body: settings: index.number_of_replicas: 0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml index 90d4080e46379..55a7fe690eb1f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml @@ -44,14 +44,11 @@ target: "copy-settings-target" wait_for_active_shards: 1 master_timeout: 10s - copy_settings: true body: settings: index.number_of_replicas: 0 index.number_of_shards: 2 index.merge.scheduler.max_thread_count: 2 - warnings: - - "parameter [copy_settings] is deprecated and will be removed in 8.0.0" - do: @@ -102,7 +99,6 @@ target: "explicit-no-copy-settings-target" wait_for_active_shards: 1 master_timeout: 10s - copy_settings: false body: settings: index.number_of_replicas: 0 diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java index 60c481e59878f..3d0158cf95f0f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java @@ -19,15 +19,10 @@ package org.elasticsearch.rest.action.admin.indices; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.Booleans; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -37,8 +32,6 @@ import java.io.IOException; public abstract class RestResizeHandler extends BaseRestHandler { - private static final Logger logger = LogManager.getLogger(RestResizeHandler.class); - private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger); RestResizeHandler(final Settings settings) { super(settings); @@ -53,24 +46,6 @@ public abstract class RestResizeHandler extends BaseRestHandler { public final RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final ResizeRequest resizeRequest = new ResizeRequest(request.param("target"), request.param("index")); resizeRequest.setResizeType(getResizeType()); - // copy_settings should be removed in Elasticsearch 8.0.0; cf. https://github.com/elastic/elasticsearch/issues/28347 - assert Version.CURRENT.major < 8; - final String rawCopySettings = request.param("copy_settings"); - final Boolean copySettings; - if (rawCopySettings == null) { - copySettings = resizeRequest.getCopySettings(); - } else { - if (rawCopySettings.isEmpty()) { - copySettings = true; - } else { - copySettings = Booleans.parseBoolean(rawCopySettings); - if (copySettings == false) { - throw new IllegalArgumentException("parameter [copy_settings] can not be explicitly set to [false]"); - } - } - deprecationLogger.deprecated("parameter [copy_settings] is deprecated and will be removed in 8.0.0"); - } - resizeRequest.setCopySettings(copySettings); request.applyContentParser(resizeRequest::fromXContent); resizeRequest.timeout(request.paramAsTime("timeout", resizeRequest.timeout())); resizeRequest.masterNodeTimeout(request.paramAsTime("master_timeout", resizeRequest.masterNodeTimeout())); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandlerTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandlerTests.java deleted file mode 100644 index b9da177b0b8fa..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandlerTests.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.FakeRestRequest; - -import java.io.IOException; -import java.util.Collections; -import java.util.Locale; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasToString; -import static org.mockito.Mockito.mock; - -public class RestResizeHandlerTests extends ESTestCase { - - public void testShrinkCopySettingsDeprecated() throws IOException { - final RestResizeHandler.RestShrinkIndexAction handler = - new RestResizeHandler.RestShrinkIndexAction(Settings.EMPTY, mock(RestController.class)); - for (final String copySettings : new String[]{null, "", "true", "false"}) { - runTestResizeCopySettingsDeprecated(handler, "shrink", copySettings); - } - } - - public void testSplitCopySettingsDeprecated() throws IOException { - final RestResizeHandler.RestSplitIndexAction handler = - new RestResizeHandler.RestSplitIndexAction(Settings.EMPTY, mock(RestController.class)); - for (final String copySettings : new String[]{null, "", "true", "false"}) { - runTestResizeCopySettingsDeprecated(handler, "split", copySettings); - } - } - - private void runTestResizeCopySettingsDeprecated( - final RestResizeHandler handler, final String resizeOperation, final String copySettings) throws IOException { - final FakeRestRequest.Builder builder = - new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) - .withParams(Collections.singletonMap("copy_settings", copySettings)) - .withPath(String.format(Locale.ROOT, "source/_%s/target", resizeOperation)); - if (copySettings != null) { - builder.withParams(Collections.singletonMap("copy_settings", copySettings)); - } - final FakeRestRequest request = builder.build(); - if ("false".equals(copySettings)) { - final IllegalArgumentException e = - expectThrows(IllegalArgumentException.class, () -> handler.prepareRequest(request, mock(NodeClient.class))); - assertThat(e, hasToString(containsString("parameter [copy_settings] can not be explicitly set to [false]"))); - } else { - handler.prepareRequest(request, mock(NodeClient.class)); - if ("".equals(copySettings) || "true".equals(copySettings)) { - assertWarnings("parameter [copy_settings] is deprecated and will be removed in 8.0.0"); - } - } - } - -} From 7e3137bf6bb0f41f359b5b0f2968d30b393a4c62 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 19:37:09 -0500 Subject: [PATCH 14/43] Revert "Use typeless APIs" This reverts commit 22eeb0c08599933c109cdff089c5f8363b2ff74c. --- .../upgrades/FullClusterRestartIT.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 070620b8e59f0..9740ff4222d7e 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -510,7 +510,7 @@ public void testRollover() throws IOException { bulk.append("{\"index\":{}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - Request bulkRequest = new Request("POST", "/" + index + "_write/_doc/_bulk"); + Request bulkRequest = new Request("POST", "/" + index + "_write/doc/_bulk"); bulkRequest.setJsonEntity(bulk.toString()); bulkRequest.addParameter("refresh", ""); bulkRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); @@ -644,12 +644,12 @@ void assertRealtimeGetWorks() throws IOException { Map hit = (Map) ((List)(XContentMapValues.extractValue("hits.hits", searchResponse))).get(0); String docId = (String) hit.get("_id"); - Request updateRequest = new Request("POST", "/" + index + "/_doc/" + docId + "/_update"); + Request updateRequest = new Request("POST", "/" + index + "/doc/" + docId + "/_update"); updateRequest.setOptions(expectWarnings(RestUpdateAction.TYPES_DEPRECATION_MESSAGE)); updateRequest.setJsonEntity("{ \"doc\" : { \"foo\": \"bar\"}}"); client().performRequest(updateRequest); - Request getRequest = new Request("GET", "/" + index + "/_doc/" + docId); + Request getRequest = new Request("GET", "/" + index + "/doc/" + docId); getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Map getRsp = entityAsMap(client().performRequest(getRequest)); Map source = (Map) getRsp.get("_source"); @@ -704,7 +704,7 @@ int extractTotalHits(Map response) { * Tests that a single document survives. Super basic smoke test. */ public void testSingleDoc() throws IOException { - String docLocation = "/" + index + "/_doc/1"; + String docLocation = "/" + index + "/doc/1"; String doc = "{\"test\": \"test\"}"; if (isRunningAgainstOldCluster()) { @@ -1016,7 +1016,7 @@ public void testSoftDeletes() throws Exception { int numDocs = between(10, 100); for (int i = 0; i < numDocs; i++) { String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject()); - Request request = new Request("POST", "/" + index + "/_doc/" + i); + Request request = new Request("POST", "/" + index + "/doc/" + i); request.setJsonEntity(doc); client().performRequest(request); if (rarely()) { @@ -1029,11 +1029,11 @@ public void testSoftDeletes() throws Exception { for (int i = 0; i < numDocs; i++) { if (randomBoolean()) { String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v2").endObject()); - Request request = new Request("POST", "/" + index + "/_doc/" + i); + Request request = new Request("POST", "/" + index + "/doc/" + i); request.setJsonEntity(doc); client().performRequest(request); } else if (randomBoolean()) { - client().performRequest(new Request("DELETE", "/" + index + "/_doc/" + i)); + client().performRequest(new Request("DELETE", "/" + index + "/doc/" + i)); liveDocs--; } } @@ -1103,7 +1103,7 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b bulk.append("{\"index\":{\"_id\":\"").append(count + i).append("\"}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/_doc/_bulk"); + Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/doc/_bulk"); writeToRestoredRequest.addParameter("refresh", "true"); writeToRestoredRequest.setJsonEntity(bulk.toString()); writeToRestoredRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); @@ -1166,7 +1166,7 @@ private void indexRandomDocuments(int count, boolean flushAllowed, boolean saveI logger.info("Indexing {} random documents", count); for (int i = 0; i < count; i++) { logger.debug("Indexing document [{}]", i); - Request createDocument = new Request("POST", "/" + index + "/_doc/" + i); + Request createDocument = new Request("POST", "/" + index + "/doc/" + i); createDocument.setJsonEntity(Strings.toString(docSupplier.apply(i))); client().performRequest(createDocument); if (rarely()) { @@ -1191,14 +1191,14 @@ private void saveInfoDocument(String type, String value) throws IOException { infoDoc.field("value", value); infoDoc.endObject(); // Only create the first version so we know how many documents are created when the index is first created - Request request = new Request("PUT", "/info/_doc/" + index + "_" + type); + Request request = new Request("PUT", "/info/doc/" + index + "_" + type); request.addParameter("op_type", "create"); request.setJsonEntity(Strings.toString(infoDoc)); client().performRequest(request); } private String loadInfoDocument(String type) throws IOException { - Request request = new Request("GET", "/info/_doc/" + index + "_" + type); + Request request = new Request("GET", "/info/doc/" + index + "_" + type); request.addParameter("filter_path", "_source"); request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); String doc = toStr(client().performRequest(request)); From 72e8ac682db7c19ef54fea2e0f378a51404bb6f4 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 21:39:11 -0500 Subject: [PATCH 15/43] Some full cluster restart tests passing --- .../upgrades/FullClusterRestartIT.java | 238 ++++++++---------- .../upgrades/QueryBuilderBWCIT.java | 12 +- .../AbstractFullClusterRestartTestCase.java | 4 + 3 files changed, 123 insertions(+), 131 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 9740ff4222d7e..4065b44ef93e4 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.rest.action.document.RestBulkAction; import org.elasticsearch.rest.action.document.RestGetAction; +import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.rest.action.document.RestUpdateAction; import org.elasticsearch.rest.action.search.RestExplainAction; import org.elasticsearch.test.NotEqualMessageBuilder; @@ -80,15 +81,20 @@ */ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { private final boolean supportsLenientBooleans = getOldClusterVersion().before(Version.V_6_0_0_alpha1); - private static final Version VERSION_5_1_0_UNRELEASED = Version.fromString("5.1.0"); private String index; + private String type; @Before public void setIndex() throws IOException { index = getTestName().toLowerCase(Locale.ROOT); } + @Before + public void setType() { + type = getOldClusterVersion().before(Version.V_7_0_0) ? "doc" : "_doc"; + } + public void testSearch() throws Exception { int count; if (isRunningAgainstOldCluster()) { @@ -102,7 +108,9 @@ public void testSearch() throws Exception { } { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } mappingsAndSettings.startObject("properties"); { mappingsAndSettings.startObject("string"); @@ -121,7 +129,9 @@ public void testSearch() throws Exception { mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); @@ -135,17 +145,20 @@ public void testSearch() throws Exception { count = randomIntBetween(2000, 3000); byte[] randomByteArray = new byte[16]; random().nextBytes(randomByteArray); - indexRandomDocuments(count, true, true, i -> { - return JsonXContent.contentBuilder().startObject() - .field("string", randomAlphaOfLength(10)) - .field("int", randomInt(100)) - .field("float", randomFloat()) - // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct - .field("bool", i > 0 && supportsLenientBooleans ? randomLenientBoolean() : randomBoolean()) - .field("field.with.dots", randomAlphaOfLength(10)) - .field("binary", Base64.getEncoder().encodeToString(randomByteArray)) - .endObject(); - }); + indexRandomDocuments( + count, + true, + true, + i -> JsonXContent.contentBuilder().startObject() + .field("string", randomAlphaOfLength(10)) + .field("int", randomInt(100)) + .field("float", randomFloat()) + // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct + .field("bool", i > 0 && supportsLenientBooleans ? randomLenientBoolean() : randomBoolean()) + .field("field.with.dots", randomAlphaOfLength(10)) + .field("binary", Base64.getEncoder().encodeToString(randomByteArray)) + .endObject() + ); refresh(); } else { count = countOfIndexedRandomDocuments(); @@ -155,7 +168,7 @@ public void testSearch() throws Exception { assertBasicSearchWorks(count); assertAllSearchWorks(count); assertBasicAggregationWorks(); - assertRealtimeGetWorks(); + assertRealtimeGetWorks(type); assertStoredBinaryFields(count); } @@ -171,7 +184,9 @@ public void testNewReplicasWork() throws Exception { } { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } mappingsAndSettings.startObject("properties"); { mappingsAndSettings.startObject("field"); @@ -179,7 +194,9 @@ public void testNewReplicasWork() throws Exception { mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); @@ -191,11 +208,8 @@ public void testNewReplicasWork() throws Exception { client().performRequest(createIndex); int numDocs = randomIntBetween(2000, 3000); - indexRandomDocuments(numDocs, true, false, i -> { - return JsonXContent.contentBuilder().startObject() - .field("field", "value") - .endObject(); - }); + indexRandomDocuments( + numDocs, true, false, i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()); logger.info("Refreshing [{}]", index); client().performRequest(new Request("POST", "/" + index + "/_refresh")); } else { @@ -225,76 +239,6 @@ public void testNewReplicasWork() throws Exception { } } - /** - * Search on an alias that contains illegal characters that would prevent it from being created after 5.1.0. It should still be - * search-able though. - */ - public void testAliasWithBadName() throws Exception { - assumeTrue("Can only test bad alias name if old cluster is on 5.1.0 or before", - getOldClusterVersion().before(VERSION_5_1_0_UNRELEASED)); - - int count; - if (isRunningAgainstOldCluster()) { - XContentBuilder mappingsAndSettings = jsonBuilder(); - mappingsAndSettings.startObject(); - { - mappingsAndSettings.startObject("settings"); - mappingsAndSettings.field("number_of_shards", 1); - mappingsAndSettings.field("number_of_replicas", 0); - mappingsAndSettings.endObject(); - } - { - mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); - mappingsAndSettings.startObject("properties"); - { - mappingsAndSettings.startObject("key"); - mappingsAndSettings.field("type", "keyword"); - mappingsAndSettings.endObject(); - } - mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); - } - mappingsAndSettings.endObject(); - Request createIndex = new Request("PUT", "/" + index); - createIndex.setJsonEntity(Strings.toString(mappingsAndSettings)); - client().performRequest(createIndex); - - String aliasName = "%23" + index; // %23 == # - client().performRequest(new Request("PUT", "/" + index + "/_alias/" + aliasName)); - Response response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + aliasName)); - assertEquals(200, response.getStatusLine().getStatusCode()); - - count = randomIntBetween(32, 128); - indexRandomDocuments(count, true, true, i -> { - return JsonXContent.contentBuilder().startObject() - .field("key", "value") - .endObject(); - }); - refresh(); - } else { - count = countOfIndexedRandomDocuments(); - } - - Request request = new Request("GET", "/_cluster/state"); - request.addParameter("metric", "metadata"); - logger.error("clusterState=" + entityAsMap(client().performRequest(request))); - // We can read from the alias just like we can read from the index. - String aliasName = "%23" + index; // %23 == # - Map searchRsp = entityAsMap(client().performRequest(new Request("GET", "/" + aliasName + "/_search"))); - int totalHits = extractTotalHits(searchRsp); - assertEquals(count, totalHits); - if (isRunningAgainstOldCluster() == false) { - // We can remove the alias. - Response response = client().performRequest(new Request("DELETE", "/" + index + "/_alias/" + aliasName)); - assertEquals(200, response.getStatusLine().getStatusCode()); - // and check that it is gone: - response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + aliasName)); - assertEquals(404, response.getStatusLine().getStatusCode()); - } - } - public void testClusterState() throws Exception { if (isRunningAgainstOldCluster()) { XContentBuilder mappingsAndSettings = jsonBuilder(); @@ -348,15 +292,25 @@ public void testShrink() throws IOException { mappingsAndSettings.startObject(); { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } mappingsAndSettings.startObject("properties"); { mappingsAndSettings.startObject("field"); mappingsAndSettings.field("type", "text"); mappingsAndSettings.endObject(); } + if (isRunningAgainstAncientCluster() == false) { + // the default number of shards is now one so we have to set the number of shards to be more than one explicitly + mappingsAndSettings.startObject("settings"); + mappingsAndSettings.field("index.number_of_shards", 5); + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); @@ -364,15 +318,13 @@ public void testShrink() throws IOException { createIndex.setJsonEntity(Strings.toString(mappingsAndSettings)); RequestOptions.Builder options = createIndex.getOptions().toBuilder(); options.setWarningsHandler(WarningsHandler.PERMISSIVE); + expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE); createIndex.setOptions(options); client().performRequest(createIndex); numDocs = randomIntBetween(512, 1024); - indexRandomDocuments(numDocs, true, true, i -> { - return JsonXContent.contentBuilder().startObject() - .field("field", "value") - .endObject(); - }); + indexRandomDocuments( + numDocs, true, true, i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()); ensureGreen(index); // wait for source index to be available on both nodes before starting shrink @@ -419,7 +371,9 @@ public void testShrinkAfterUpgrade() throws IOException { mappingsAndSettings.startObject(); { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } mappingsAndSettings.startObject("properties"); { mappingsAndSettings.startObject("field"); @@ -427,7 +381,9 @@ public void testShrinkAfterUpgrade() throws IOException { mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); @@ -439,11 +395,12 @@ public void testShrinkAfterUpgrade() throws IOException { client().performRequest(createIndex); numDocs = randomIntBetween(512, 1024); - indexRandomDocuments(numDocs, true, true, i -> { - return JsonXContent.contentBuilder().startObject() - .field("field", "value") - .endObject(); - }); + indexRandomDocuments( + numDocs, + true, + true, + i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject() + ); } else { ensureGreen(index); // wait for source index to be available on both nodes before starting shrink @@ -510,7 +467,7 @@ public void testRollover() throws IOException { bulk.append("{\"index\":{}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - Request bulkRequest = new Request("POST", "/" + index + "_write/doc/_bulk"); + Request bulkRequest = new Request("POST", "/" + index + "_write/" + type + "/_bulk"); bulkRequest.setJsonEntity(bulk.toString()); bulkRequest.addParameter("refresh", ""); bulkRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); @@ -633,7 +590,7 @@ void assertBasicAggregationWorks() throws IOException { assertTotalHits(termsCount, boolTerms); } - void assertRealtimeGetWorks() throws IOException { + void assertRealtimeGetWorks(final String typeName) throws IOException { Request disableAutoRefresh = new Request("PUT", "/" + index + "/_settings"); disableAutoRefresh.setJsonEntity("{ \"index\": { \"refresh_interval\" : -1 }}"); client().performRequest(disableAutoRefresh); @@ -644,12 +601,12 @@ void assertRealtimeGetWorks() throws IOException { Map hit = (Map) ((List)(XContentMapValues.extractValue("hits.hits", searchResponse))).get(0); String docId = (String) hit.get("_id"); - Request updateRequest = new Request("POST", "/" + index + "/doc/" + docId + "/_update"); + Request updateRequest = new Request("POST", "/" + index + "/" + typeName + "/" + docId + "/_update"); updateRequest.setOptions(expectWarnings(RestUpdateAction.TYPES_DEPRECATION_MESSAGE)); updateRequest.setJsonEntity("{ \"doc\" : { \"foo\": \"bar\"}}"); client().performRequest(updateRequest); - Request getRequest = new Request("GET", "/" + index + "/doc/" + docId); + Request getRequest = new Request("GET", "/" + index + "/" + typeName + "/" + docId); getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Map getRsp = entityAsMap(client().performRequest(getRequest)); Map source = (Map) getRsp.get("_source"); @@ -704,7 +661,7 @@ int extractTotalHits(Map response) { * Tests that a single document survives. Super basic smoke test. */ public void testSingleDoc() throws IOException { - String docLocation = "/" + index + "/doc/1"; + String docLocation = "/" + index + "/" + type + "/1"; String doc = "{\"test\": \"test\"}"; if (isRunningAgainstOldCluster()) { @@ -779,8 +736,12 @@ public void testRecovery() throws Exception { } if (shouldHaveTranslog) { // Update a few documents so we are sure to have a translog - indexRandomDocuments(count / 10, false /* Flushing here would invalidate the whole thing....*/, false, - i -> jsonBuilder().startObject().field("field", "value").endObject()); + indexRandomDocuments( + count / 10, + false, // flushing here would invalidate the whole thing + false, + i -> jsonBuilder().startObject().field("field", "value").endObject() + ); } saveInfoDocument("should_have_translog", Boolean.toString(shouldHaveTranslog)); } else { @@ -894,13 +855,19 @@ public void testSnapshotRestore() throws IOException { } templateBuilder.endObject(); templateBuilder.startObject("mappings"); { - templateBuilder.startObject("doc"); { - templateBuilder.startObject("_source"); { + if (isRunningAgainstAncientCluster()) { + templateBuilder.startObject(type); + } + { + templateBuilder.startObject("_source"); + { templateBuilder.field("enabled", true); } templateBuilder.endObject(); } - templateBuilder.endObject(); + if (isRunningAgainstAncientCluster()) { + templateBuilder.endObject(); + } } templateBuilder.endObject(); templateBuilder.startObject("aliases"); { @@ -1016,7 +983,10 @@ public void testSoftDeletes() throws Exception { int numDocs = between(10, 100); for (int i = 0; i < numDocs; i++) { String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject()); - Request request = new Request("POST", "/" + index + "/doc/" + i); + Request request = new Request("POST", "/" + index + "/" + type + "/" + i); + if (isRunningAgainstAncientCluster() == false) { + request.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); + } request.setJsonEntity(doc); client().performRequest(request); if (rarely()) { @@ -1029,11 +999,11 @@ public void testSoftDeletes() throws Exception { for (int i = 0; i < numDocs; i++) { if (randomBoolean()) { String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v2").endObject()); - Request request = new Request("POST", "/" + index + "/doc/" + i); + Request request = new Request("POST", "/" + index + "/" + type + "/" + i); request.setJsonEntity(doc); client().performRequest(request); } else if (randomBoolean()) { - client().performRequest(new Request("DELETE", "/" + index + "/doc/" + i)); + client().performRequest(new Request("DELETE", "/" + index + "/" + type + "/" + i)); liveDocs--; } } @@ -1046,7 +1016,7 @@ public void testSoftDeletes() throws Exception { } } - private void checkSnapshot(String snapshotName, int count, Version tookOnVersion) throws IOException { + private void checkSnapshot(final String snapshotName, final int count, final Version tookOnVersion) throws IOException { // Check the snapshot metadata, especially the version Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName); Map listSnapshotResponse = entityAsMap(client().performRequest(listSnapshotRequest)); @@ -1103,7 +1073,7 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b bulk.append("{\"index\":{\"_id\":\"").append(count + i).append("\"}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/doc/_bulk"); + Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/" + type + "/_bulk"); writeToRestoredRequest.addParameter("refresh", "true"); writeToRestoredRequest.setJsonEntity(bulk.toString()); writeToRestoredRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); @@ -1145,7 +1115,7 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b expectedTemplate.put("index_patterns", singletonList("evil_*")); } expectedTemplate.put("settings", singletonMap("index", singletonMap("number_of_shards", "1"))); - expectedTemplate.put("mappings", singletonMap("doc", singletonMap("_source", singletonMap("enabled", true)))); + expectedTemplate.put("mappings", singletonMap(type, singletonMap("_source", singletonMap("enabled", true)))); expectedTemplate.put("order", 0); Map aliases = new HashMap<>(); aliases.put("alias1", emptyMap()); @@ -1161,12 +1131,19 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b // TODO tests for upgrades after shrink. We've had trouble with shrink in the past. - private void indexRandomDocuments(int count, boolean flushAllowed, boolean saveInfo, - CheckedFunction docSupplier) throws IOException { + private void indexRandomDocuments( + final int count, + final boolean flushAllowed, + final boolean saveInfo, + final CheckedFunction docSupplier) + throws IOException { logger.info("Indexing {} random documents", count); for (int i = 0; i < count; i++) { logger.debug("Indexing document [{}]", i); - Request createDocument = new Request("POST", "/" + index + "/doc/" + i); + Request createDocument = new Request("POST", "/" + index + "/" + type + "/" + i); + if (isRunningAgainstAncientCluster() == false) { + createDocument.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); + } createDocument.setJsonEntity(Strings.toString(docSupplier.apply(i))); client().performRequest(createDocument); if (rarely()) { @@ -1191,16 +1168,21 @@ private void saveInfoDocument(String type, String value) throws IOException { infoDoc.field("value", value); infoDoc.endObject(); // Only create the first version so we know how many documents are created when the index is first created - Request request = new Request("PUT", "/info/doc/" + index + "_" + type); + Request request = new Request("PUT", "/info/" + this.type + "/" + index + "_" + type); request.addParameter("op_type", "create"); request.setJsonEntity(Strings.toString(infoDoc)); + if (isRunningAgainstAncientCluster() == false) { + request.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); + } client().performRequest(request); } private String loadInfoDocument(String type) throws IOException { - Request request = new Request("GET", "/info/doc/" + index + "_" + type); + Request request = new Request("GET", "/info/" + this.type + "/" + index + "_" + type); request.addParameter("filter_path", "_source"); - request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + if (isRunningAgainstAncientCluster() == false) { + request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + } String doc = toStr(client().performRequest(request)); Matcher m = Pattern.compile("\"value\":\"(.+)\"").matcher(doc); assertTrue(doc, m.find()); diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index f22b1b44c0763..d6cc9f078e0ed 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.upgrades; import org.apache.http.util.EntityUtils; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -145,6 +146,7 @@ private static void addCandidate(String querySource, QueryBuilder expectedQb) { } public void testQueryBuilderBWC() throws Exception { + final String type = getOldClusterVersion().before(Version.V_7_0_0) ? "doc" : "_doc"; String index = "queries"; if (isRunningAgainstOldCluster()) { XContentBuilder mappingsAndSettings = jsonBuilder(); @@ -157,7 +159,9 @@ public void testQueryBuilderBWC() throws Exception { } { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } mappingsAndSettings.startObject("properties"); { mappingsAndSettings.startObject("query"); @@ -176,7 +180,9 @@ public void testQueryBuilderBWC() throws Exception { } mappingsAndSettings.endObject(); mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } } mappingsAndSettings.endObject(); Request request = new Request("PUT", "/" + index); @@ -188,7 +194,7 @@ public void testQueryBuilderBWC() throws Exception { assertEquals(200, rsp.getStatusLine().getStatusCode()); for (int i = 0; i < CANDIDATES.size(); i++) { - request = new Request("PUT", "/" + index + "/doc/" + Integer.toString(i)); + request = new Request("PUT", "/" + index + "/" + type + "/" + Integer.toString(i)); request.setJsonEntity((String) CANDIDATES.get(i)[0]); rsp = client().performRequest(request); assertEquals(201, rsp.getStatusLine().getStatusCode()); diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index 508949b561fb4..97d29181c59d3 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -63,6 +63,10 @@ public final boolean isRunningAgainstOldCluster() { private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); + public final boolean isRunningAgainstAncientCluster() { + return oldClusterVersion.before(Version.V_7_0_0); + } + public final Version getOldClusterVersion() { return oldClusterVersion; } From 93b298ad55bd96578662ff0a64bc1e8d221605ff Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 6 Feb 2019 23:51:14 -0500 Subject: [PATCH 16/43] More fixes --- distribution/docker/src/docker/Dockerfile | 4 +- .../upgrades/FullClusterRestartIT.java | 65 ++++++++++++------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 37bbfe6688597..c202fa78668f5 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -30,13 +30,13 @@ RUN groupadd -g 1000 elasticsearch && \ WORKDIR /usr/share/elasticsearch -COPY ${elasticsearch} /opt/ +${source_elasticsearch} + RUN tar zxf /opt/${elasticsearch} --strip-components=1 RUN mkdir -p config data logs RUN chmod 0775 config data logs COPY config/elasticsearch.yml config/log4j2.properties config/ - ################################################################################ # Build stage 1 (the actual elasticsearch image): # Copy elasticsearch from stage 0 diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 4065b44ef93e4..940b62042495a 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -292,26 +292,32 @@ public void testShrink() throws IOException { mappingsAndSettings.startObject(); { mappingsAndSettings.startObject("mappings"); - if (isRunningAgainstAncientCluster()) { - mappingsAndSettings.startObject(type); - } - mappingsAndSettings.startObject("properties"); { - mappingsAndSettings.startObject("field"); - mappingsAndSettings.field("type", "text"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } + mappingsAndSettings.startObject("properties"); + { + mappingsAndSettings.startObject("field"); + { + mappingsAndSettings.field("type", "text"); + } + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } } + mappingsAndSettings.endObject(); if (isRunningAgainstAncientCluster() == false) { // the default number of shards is now one so we have to set the number of shards to be more than one explicitly mappingsAndSettings.startObject("settings"); - mappingsAndSettings.field("index.number_of_shards", 5); - mappingsAndSettings.endObject(); - } - mappingsAndSettings.endObject(); - if (isRunningAgainstAncientCluster()) { + { + mappingsAndSettings.field("index.number_of_shards", 5); + } mappingsAndSettings.endObject(); } - mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); Request createIndex = new Request("PUT", "/" + index); @@ -333,7 +339,7 @@ public void testShrink() throws IOException { client().performRequest(updateSettingsRequest); Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex); - if (getOldClusterVersion().onOrAfter(Version.V_6_4_0)) { + if (getOldClusterVersion().onOrAfter(Version.V_6_4_0) && getOldClusterVersion().before(Version.V_7_0_0)) { shrinkIndexRequest.addParameter("copy_settings", "true"); } shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}"); @@ -371,20 +377,30 @@ public void testShrinkAfterUpgrade() throws IOException { mappingsAndSettings.startObject(); { mappingsAndSettings.startObject("mappings"); - if (isRunningAgainstAncientCluster()) { - mappingsAndSettings.startObject(type); - } - mappingsAndSettings.startObject("properties"); { - mappingsAndSettings.startObject("field"); - mappingsAndSettings.field("type", "text"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } + mappingsAndSettings.startObject("properties"); + { + mappingsAndSettings.startObject("field"); + { + mappingsAndSettings.field("type", "text"); + } + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } } mappingsAndSettings.endObject(); - if (isRunningAgainstAncientCluster()) { + if (isRunningAgainstAncientCluster() == false) { + // the default number of shards is now one so we have to set the number of shards to be more than one explicitly + mappingsAndSettings.startObject("settings"); + mappingsAndSettings.field("index.number_of_shards", 5); mappingsAndSettings.endObject(); } - mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); Request createIndex = new Request("PUT", "/" + index); @@ -753,6 +769,7 @@ public void testRecovery() throws Exception { Request countRequest = new Request("GET", "/" + index + "/_search"); countRequest.addParameter("size", "0"); Map countResponse = entityAsMap(client().performRequest(countRequest)); + refresh(); assertTotalHits(count, countResponse); if (false == isRunningAgainstOldCluster()) { @@ -989,9 +1006,7 @@ public void testSoftDeletes() throws Exception { } request.setJsonEntity(doc); client().performRequest(request); - if (rarely()) { - refresh(); - } + refresh(); } client().performRequest(new Request("POST", "/" + index + "/_flush")); int liveDocs = numDocs; @@ -1102,7 +1117,7 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b // In 7.0, type names are no longer returned by default in get index template requests. // We therefore use the deprecated typed APIs when running against the current version. - if (isRunningAgainstOldCluster() == false) { + if (isRunningAgainstAncientCluster() == false) { getTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); } getTemplateRequest.setOptions(allowTypeRemovalWarnings()); From 7e8345d329eb9b36ea78554e6b38b25faa1a4f44 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 06:58:20 +0200 Subject: [PATCH 17/43] Fixes for rolling upgrade --- .../org/elasticsearch/upgrades/RecoveryIT.java | 6 +++++- .../test/mixed_cluster/10_basic.yml | 5 ----- .../rest-api-spec/test/old_cluster/10_basic.yml | 16 ---------------- .../test/upgraded_cluster/10_basic.yml | 13 ------------- 4 files changed, 5 insertions(+), 35 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index 3666a64896ae2..295aee8b869ff 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -300,7 +300,11 @@ public void testRecoveryWithSoftDeletes() throws Exception { if (randomBoolean()) { indexDocs(index, i, 1); // update } else if (randomBoolean()) { - client().performRequest(new Request("DELETE", index + "/test/" + i)); + if (getNodeId(v -> v.onOrAfter(Version.V_7_0_0)) == null) { + client().performRequest(new Request("DELETE", index + "/test/" + i)); + } else { + client().performRequest(new Request("DELETE", index + "/_doc/" + i)); + } } } } diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index bd07ee8a58469..375ba12a35621 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -67,8 +67,3 @@ - match: { hits.total: 1 } - match: { hits.hits.0._id: q3 } ---- -"Index with _all is available": - - do: - indices.get: - index: all-index diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index a26a3f8274d99..2672cee7cc78a 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -200,19 +200,3 @@ wait_for_completion: true task_id: $task ---- -"Create an index with _all explicitly disabled": - - skip: - features: warnings - - do: - warnings: - - "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field." - indices.create: - index: all-index - body: - mappings: - _all: - enabled: false - properties: - field: - type: text diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 63e67652127e9..78a4aac867d8f 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -125,17 +125,4 @@ task_id: $task_id - match: { task.headers.X-Opaque-Id: "Reindexing Again" } ---- -"Index with _all is available": - - do: - indices.get: - index: all-index - - - do: - indices.get_mapping: - include_type_name: false - index: all-index - - - is_true: all-index.mappings._all - - match: { all-index.mappings._all.enabled: false} From 711fd82ff5d9727aeed0dca04014468b7cc4e644 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 10:01:33 +0200 Subject: [PATCH 18/43] Fix full cluster restart tests --- .../upgrades/FullClusterRestartIT.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 940b62042495a..7f890414c1f96 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -92,7 +92,7 @@ public void setIndex() throws IOException { @Before public void setType() { - type = getOldClusterVersion().before(Version.V_7_0_0) ? "doc" : "_doc"; + type = getOldClusterVersion().before(Version.V_6_7_0) ? "doc" : "_doc"; } public void testSearch() throws Exception { @@ -623,7 +623,9 @@ void assertRealtimeGetWorks(final String typeName) throws IOException { client().performRequest(updateRequest); Request getRequest = new Request("GET", "/" + index + "/" + typeName + "/" + docId); - getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + } Map getRsp = entityAsMap(client().performRequest(getRequest)); Map source = (Map) getRsp.get("_source"); assertTrue("doc does not contain 'foo' key: " + source, source.containsKey("foo")); @@ -688,7 +690,9 @@ public void testSingleDoc() throws IOException { Request request = new Request("GET", docLocation); - request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + } assertThat(toStr(client().performRequest(request)), containsString(doc)); } @@ -841,7 +845,7 @@ public void testRecovery() throws Exception { */ public void testSnapshotRestore() throws IOException { int count; - if (isRunningAgainstOldCluster()) { + if (isRunningAgainstOldCluster() && getOldClusterVersion().major < 8) { // Create the index count = between(200, 300); indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject()); @@ -906,7 +910,7 @@ public void testSnapshotRestore() throws IOException { // In 7.0, type names are no longer expected by default in put index template requests. // We therefore use the deprecated typed APIs when running against the current version. - if (isRunningAgainstOldCluster() == false) { + if (isRunningAgainstOldCluster() == false && getOldClusterVersion().major < 7) { createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); } createTemplateRequest.setOptions(allowTypeRemovalWarnings()); @@ -1130,7 +1134,14 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b expectedTemplate.put("index_patterns", singletonList("evil_*")); } expectedTemplate.put("settings", singletonMap("index", singletonMap("number_of_shards", "1"))); - expectedTemplate.put("mappings", singletonMap(type, singletonMap("_source", singletonMap("enabled", true)))); + // We don't have the type in the response starting with 7.0, but we won't have it on old cluster after upgrade + // either so look at the response to figure out the correct assertions + if (isTypeInTemplateResponse(getTemplateResponse)) { + expectedTemplate.put("mappings", singletonMap(type, singletonMap("_source", singletonMap("enabled", true)))); + } else { + expectedTemplate.put("mappings", singletonMap("_source", singletonMap("enabled", true))); + } + expectedTemplate.put("order", 0); Map aliases = new HashMap<>(); aliases.put("alias1", emptyMap()); @@ -1140,10 +1151,18 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b if (false == expectedTemplate.equals(getTemplateResponse)) { NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); builder.compareMaps(getTemplateResponse, expectedTemplate); + logger.info("expected: {}\nactual:{}", expectedTemplate, getTemplateResponse); fail("template doesn't match:\n" + builder.toString()); } } + @SuppressWarnings("unchecked") + private boolean isTypeInTemplateResponse(Map getTemplateResponse) { + return ( (Map) ( + (Map) getTemplateResponse.getOrDefault("test_template", emptyMap()) + ).get("mappings")).get("_source") == null; + } + // TODO tests for upgrades after shrink. We've had trouble with shrink in the past. private void indexRandomDocuments( @@ -1195,7 +1214,7 @@ private void saveInfoDocument(String type, String value) throws IOException { private String loadInfoDocument(String type) throws IOException { Request request = new Request("GET", "/info/" + this.type + "/" + index + "_" + type); request.addParameter("filter_path", "_source"); - if (isRunningAgainstAncientCluster() == false) { + if (isRunningAgainstAncientCluster() == false && getOldClusterVersion().major < 7) { request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); } String doc = toStr(client().performRequest(request)); From 8b537b9ee4806d63d1488547ad0be0a2dad2fd61 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 7 Feb 2019 10:43:12 +0100 Subject: [PATCH 19/43] Comment out asserts, and remove bwc logic later #38557 --- .../java/org/elasticsearch/gateway/MetaStateService.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 7bef4d5819e45..04bd4b5341652 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -21,7 +21,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; @@ -118,7 +117,7 @@ private Tuple loadFullStateBWC() throws IOException { if (globalMetaData != null) { metaDataBuilder = MetaData.builder(globalMetaData); - assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; +// assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; } else { metaDataBuilder = MetaData.builder(); } @@ -127,7 +126,7 @@ private Tuple loadFullStateBWC() throws IOException { Tuple indexMetaDataAndGeneration = INDEX_META_DATA_FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.resolveIndexFolder(indexFolderName)); - assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; +// assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1(); long generation = indexMetaDataAndGeneration.v2(); if (indexMetaData != null) { @@ -291,4 +290,4 @@ public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) writeManifestAndCleanup(reason, manifest); cleanupGlobalState(generation); } -} \ No newline at end of file +} From ea1822efc76cb3dc5033533b6fac78dd4463ee97 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Feb 2019 09:25:37 +0000 Subject: [PATCH 20/43] Refresh before getting doc count --- .../java/org/elasticsearch/upgrades/FullClusterRestartIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 3a35dae3d8ca4..19c7a1d5ce87b 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -662,7 +662,7 @@ static void assertNoFailures(Map response) { void assertTotalHits(int expectedTotalHits, Map response) { int actualTotalHits = extractTotalHits(response); - assertEquals(expectedTotalHits, actualTotalHits); + assertEquals(response.toString(), expectedTotalHits, actualTotalHits); } int extractTotalHits(Map response) { @@ -770,8 +770,8 @@ public void testRecovery() throws Exception { // Count the documents in the index to make sure we have as many as we put there Request countRequest = new Request("GET", "/" + index + "/_search"); countRequest.addParameter("size", "0"); - Map countResponse = entityAsMap(client().performRequest(countRequest)); refresh(); + Map countResponse = entityAsMap(client().performRequest(countRequest)); assertTotalHits(count, countResponse); if (false == isRunningAgainstOldCluster()) { From 6b87785957214cabc6595f5ce6a1cca4eacebcec Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 11:49:46 +0200 Subject: [PATCH 21/43] Mute version assertions Tracking #38556 --- .../action/resync/ResyncReplicationRequest.java | 3 ++- .../cluster/metadata/MetaDataCreateIndexService.java | 3 ++- .../java/org/elasticsearch/gateway/MetaStateService.java | 6 ++++-- .../org/elasticsearch/transport/RemoteClusterAware.java | 3 ++- .../org/elasticsearch/transport/RemoteClusterService.java | 3 ++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java b/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java index a53766af7cf52..2ec0e9200d436 100644 --- a/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java +++ b/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java @@ -66,7 +66,8 @@ public Translog.Operation[] getOperations() { @Override public void readFrom(final StreamInput in) throws IOException { - assert Version.CURRENT.major <= 7; + // TODO: https://github.com/elastic/elasticsearch/issues/38556 + //assert Version.CURRENT.major <= 7; if (in.getVersion().equals(Version.V_6_0_0)) { /* * Resync replication request serialization was broken in 6.0.0 due to the elements of the stream not being prefixed with a diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 838b1e2547204..2e18e5aeae50a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -590,7 +590,8 @@ public ClusterState execute(ClusterState currentState) throws Exception { static int getNumberOfShards(final Settings.Builder indexSettingsBuilder) { // TODO: this logic can be removed when the current major version is 8 - assert Version.CURRENT.major == 7; + // TODO: https://github.com/elastic/elasticsearch/issues/38556 + // assert Version.CURRENT.major == 7; final int numberOfShards; final Version indexVersionCreated = Version.fromId(Integer.parseInt(indexSettingsBuilder.get(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey()))); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 04bd4b5341652..d67cdccb9a09b 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -117,7 +117,8 @@ private Tuple loadFullStateBWC() throws IOException { if (globalMetaData != null) { metaDataBuilder = MetaData.builder(globalMetaData); -// assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; + // TODO https://github.com/elastic/elasticsearch/issues/38556 + // assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; } else { metaDataBuilder = MetaData.builder(); } @@ -126,7 +127,8 @@ private Tuple loadFullStateBWC() throws IOException { Tuple indexMetaDataAndGeneration = INDEX_META_DATA_FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.resolveIndexFolder(indexFolderName)); -// assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; + // TODO https://github.com/elastic/elasticsearch/issues/38556 + // assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1(); long generation = indexMetaDataAndGeneration.v2(); if (indexMetaData != null) { diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 9b9243b612b74..ff06e59d4f729 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -57,7 +57,8 @@ public abstract class RemoteClusterAware { static { // remove search.remote.* settings in 8.0.0 - assert Version.CURRENT.major < 8; + // TODO https://github.com/elastic/elasticsearch/issues/38556 + // assert Version.CURRENT.major < 8; } public static final Setting.AffixSetting> SEARCH_REMOTE_CLUSTERS_SEEDS = diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index 009ee48dd8a99..a126337aa0e54 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -69,7 +69,8 @@ public final class RemoteClusterService extends RemoteClusterAware implements Cl static { // remove search.remote.* settings in 8.0.0 - assert Version.CURRENT.major < 8; + // TODO + // assert Version.CURRENT.major < 8; } public static final Setting SEARCH_REMOTE_CONNECTIONS_PER_CLUSTER = From d9ec29400e6a0f98b162badc4fb175314f11ee7b Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 12:06:23 +0200 Subject: [PATCH 22/43] :server:unitTest passing --- .../cluster/metadata/MetaDataIndexUpgradeServiceTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java index c1e341fd5bc2f..827680cca1b71 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java @@ -89,7 +89,10 @@ public void testFailUpgrade() { new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.emptyList()); Version minCompat = Version.CURRENT.minimumIndexCompatibilityVersion(); - Version indexUpgraded = VersionUtils.randomVersionBetween(random(), minCompat, VersionUtils.getPreviousVersion(Version.CURRENT)); + Version indexUpgraded = VersionUtils.randomVersionBetween(random(), + minCompat, + Version.max(minCompat, VersionUtils.getPreviousVersion(Version.CURRENT)) + ); Version indexCreated = Version.fromString((minCompat.major - 1) + "." + randomInt(5) + "." + randomInt(5)); final IndexMetaData metaData = newIndexMeta("foo", Settings.builder() .put(IndexMetaData.SETTING_VERSION_UPGRADED, indexUpgraded) From b468e42a69e7fd1780eb221739ce5255dde0dc51 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 12:21:49 +0200 Subject: [PATCH 23/43] Fix full cluster restart --- .../AbstractFullClusterRestartTestCase.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index 97d29181c59d3..916f97ba5b89c 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -36,27 +36,6 @@ public abstract class AbstractFullClusterRestartTestCase extends ESRestTestCase private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster")); - @Before - public void init() throws IOException { - assertThat("we don't need this branch if we aren't compatible with 6.0", - Version.CURRENT.minimumIndexCompatibilityVersion().onOrBefore(Version.V_6_0_0), equalTo(true)); - if (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_0_0)) { - XContentBuilder template = jsonBuilder(); - template.startObject(); - { - template.field("index_patterns", "*"); - template.field("order", "0"); - template.startObject("settings"); - template.field("number_of_shards", 5); - template.endObject(); - } - template.endObject(); - Request createTemplate = new Request("PUT", "/_template/template"); - createTemplate.setJsonEntity(Strings.toString(template)); - client().performRequest(createTemplate); - } - } - public final boolean isRunningAgainstOldCluster() { return runningAgainstOldCluster; } From 29c4a2221fba91aa2855e8871d1ebe939fc2be53 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 7 Feb 2019 14:41:36 +0100 Subject: [PATCH 24/43] removed no longer needed bwc code. --- .../elasticsearch/upgrades/IndexingIT.java | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index c80218c50ebe9..96ebaedadce93 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -19,20 +19,15 @@ package org.elasticsearch.upgrades; import org.apache.http.util.EntityUtils; -import org.elasticsearch.common.Booleans; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.action.document.RestBulkAction; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.rest.action.document.RestBulkAction; import java.io.IOException; import java.nio.charset.StandardCharsets; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; -import static org.hamcrest.Matchers.equalTo; /** * Basic test that indexed documents survive the rolling restart. See @@ -68,26 +63,6 @@ public void testIndexing() throws IOException { } if (CLUSTER_TYPE == ClusterType.OLD) { - { - Version minimumIndexCompatibilityVersion = Version.CURRENT.minimumIndexCompatibilityVersion(); - assertThat("this branch is not needed if we aren't compatible with 6.0", - minimumIndexCompatibilityVersion.onOrBefore(Version.V_6_0_0), equalTo(true)); - if (minimumIndexCompatibilityVersion.before(Version.V_7_0_0)) { - XContentBuilder template = jsonBuilder(); - template.startObject(); - { - template.field("index_patterns", "*"); - template.startObject("settings"); - template.field("number_of_shards", 5); - template.endObject(); - } - template.endObject(); - Request createTemplate = new Request("PUT", "/_template/template"); - createTemplate.setJsonEntity(Strings.toString(template)); - client().performRequest(createTemplate); - } - } - Request createTestIndex = new Request("PUT", "/test_index"); createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}"); client().performRequest(createTestIndex); From e5b928d1be26c608f00b940b1e3e39fd37fb7cb4 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 7 Feb 2019 16:18:42 +0200 Subject: [PATCH 25/43] Leftover rest test from #38553 --- .../test/indices.shrink/30_copy_settings.yml | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml index 4cf77fe85caf0..3add4b100d812 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml @@ -61,41 +61,3 @@ - match: { copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" } - match: { copy-settings-target.settings.index.blocks.write: "true" } - match: { copy-settings-target.settings.index.routing.allocation.include._id: $master } - - # now we do a actual shrink and copy settings (by default) - - do: - indices.shrink: - index: "source" - target: "default-copy-settings-target" - wait_for_active_shards: 1 - master_timeout: 10s - body: - settings: - index.number_of_replicas: 0 - index.merge.scheduler.max_thread_count: 2 - - - do: - cluster.health: - wait_for_status: green - - - do: - indices.get_settings: - index: "default-copy-settings-target" - - # settings should be copied - - match: { default-copy-settings-target.settings.index.merge.scheduler.max_merge_count: "4" } - - match: { default-copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" } - - match: { default-copy-settings-target.settings.index.blocks.write: "true" } - - match: { default-copy-settings-target.settings.index.routing.allocation.include._id: $master } - - # now we do a actual shrink and try to set no copy settings - - do: - catch: /illegal_argument_exception/ - indices.shrink: - index: "source" - target: "explicit-no-copy-settings-target" - wait_for_active_shards: 1 - master_timeout: 10s - body: - settings: - index.number_of_replicas: 0 From cb8b59eef55438e1e58de9ba279710a106c0fabc Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 7 Feb 2019 16:44:55 +0200 Subject: [PATCH 26/43] Left over from e5b928d1be26c608f00b940b1e3e3 --- .../test/indices.split/30_copy_settings.yml | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml index 55a7fe690eb1f..8cf932b1c1159 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml @@ -50,7 +50,6 @@ index.number_of_shards: 2 index.merge.scheduler.max_thread_count: 2 - - do: cluster.health: wait_for_status: green @@ -64,41 +63,3 @@ - match: { copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" } - match: { copy-settings-target.settings.index.blocks.write: "true" } - match: { copy-settings-target.settings.index.routing.allocation.include._id: $master } - - # now we do a actual shrink and copy settings (by default) - - do: - indices.split: - index: "source" - target: "default-copy-settings-target" - wait_for_active_shards: 1 - master_timeout: 10s - body: - settings: - index.number_of_replicas: 0 - index.number_of_shards: 2 - index.merge.scheduler.max_thread_count: 2 - - - do: - cluster.health: - wait_for_status: green - - - do: - indices.get_settings: - index: "default-copy-settings-target" - - # settings should be copied - - match: { default-copy-settings-target.settings.index.merge.scheduler.max_merge_count: "4" } - - match: { default-copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" } - - match: { default-copy-settings-target.settings.index.blocks.write: "true" } - - match: { default-copy-settings-target.settings.index.routing.allocation.include._id: $master } - - - do: - catch: /illegal_argument_exception/ - indices.split: - index: "source" - target: "explicit-no-copy-settings-target" - wait_for_active_shards: 1 - master_timeout: 10s - body: - settings: - index.number_of_replicas: 0 From 1601d8e0ff64b7a0f48b3b8a4220a78cf26dea9e Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 15:46:36 +0200 Subject: [PATCH 27/43] fix verify version constants --- .../qa/verify_version_constants/VerifyVersionConstantsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java index 56297eaab8857..bba6b4a8aa27f 100644 --- a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java +++ b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java @@ -37,7 +37,7 @@ public void testLuceneVersionConstant() throws IOException, ParseException { assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); final ObjectPath objectPath = ObjectPath.createFromResponse(response); final String elasticsearchVersionString = objectPath.evaluate("version.number").toString(); - final Version elasticsearchVersion = Version.fromString(elasticsearchVersionString); + final Version elasticsearchVersion = Version.fromString(elasticsearchVersionString.replace("-SNAPSHOT", "")); final String luceneVersionString = objectPath.evaluate("version.lucene_version").toString(); final org.apache.lucene.util.Version luceneVersion = org.apache.lucene.util.Version.parse(luceneVersionString); assertThat(elasticsearchVersion.luceneVersion, equalTo(luceneVersion)); From f1077d1de034d19ab224d35bfe99da69dea1d6a2 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 7 Feb 2019 08:05:08 -0500 Subject: [PATCH 28/43] Fix method --- .../upgrades/AbstractFullClusterRestartTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index 916f97ba5b89c..97e8e87e5dc4d 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -43,7 +43,7 @@ public final boolean isRunningAgainstOldCluster() { private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); public final boolean isRunningAgainstAncientCluster() { - return oldClusterVersion.before(Version.V_7_0_0); + return isRunningAgainstOldCluster() && oldClusterVersion.before(Version.V_7_0_0); } public final Version getOldClusterVersion() { From 0c5d19a0c10de0b8fa9457350d4753345910abe3 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 16:43:56 +0200 Subject: [PATCH 29/43] Fix x-pack full cluster restart --- .../xpack/restart/FullClusterRestartIT.java | 159 ++++++++++++------ 1 file changed, 104 insertions(+), 55 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index e4d96645b87b7..53658eba0cea2 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -28,10 +28,10 @@ import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition; -import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger; import org.hamcrest.Matcher; +import org.junit.Before; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -60,6 +60,13 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { + private String type; + + @Before + public void setType() { + type = getOldClusterVersion().before(Version.V_6_7_0) ? "doc" : "_doc"; + } + @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); @@ -76,7 +83,7 @@ protected Settings restClientSettings() { * Tests that a single document survives. Super basic smoke test. */ public void testSingleDoc() throws IOException { - String docLocation = "/testsingledoc/doc/1"; + String docLocation = "/testsingledoc/" + type + "/1"; String doc = "{\"test\": \"test\"}"; if (isRunningAgainstOldCluster()) { @@ -87,7 +94,9 @@ public void testSingleDoc() throws IOException { } Request getRequest = new Request("GET", docLocation); - getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + } assertThat(toStr(client().performRequest(getRequest)), containsString(doc)); } @@ -148,17 +157,21 @@ public void testSecurityNativeRealm() throws Exception { public void testWatcher() throws Exception { if (isRunningAgainstOldCluster()) { logger.info("Adding a watch on old cluster {}", getOldClusterVersion()); - Request createBwcWatch = new Request("PUT", "/_xpack/watcher/watch/bwc_watch"); + Request createBwcWatch = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_watch"); + Request createBwcThrottlePeriod = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_throttle_period"); + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0)) { + createBwcWatch.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE)); + createBwcThrottlePeriod.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE)); + } createBwcWatch.setJsonEntity(loadWatch("simple-watch.json")); client().performRequest(createBwcWatch); logger.info("Adding a watch with \"fun\" throttle periods on old cluster"); - Request createBwcThrottlePeriod = new Request("PUT", "_xpack/watcher/watch/bwc_throttle_period"); createBwcThrottlePeriod.setJsonEntity(loadWatch("throttle-period-watch.json")); client().performRequest(createBwcThrottlePeriod); logger.info("Adding a watch with \"fun\" read timeout on old cluster"); - Request createFunnyTimeout = new Request("PUT", "_xpack/watcher/watch/bwc_funny_timeout"); + Request createFunnyTimeout = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_funny_timeout"); createFunnyTimeout.setJsonEntity(loadWatch("funny-timeout-watch.json")); client().performRequest(createFunnyTimeout); @@ -246,7 +259,11 @@ public void testRollupAfterRestart() throws Exception { // index documents for the rollup job final StringBuilder bulk = new StringBuilder(); for (int i = 0; i < numDocs; i++) { - bulk.append("{\"index\":{\"_index\":\"rollup-docs\",\"_type\":\"doc\"}}\n"); + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0)) { + bulk.append("{\"index\":{\"_index\":\"rollup-docs\"}}\n"); + } else { + bulk.append("{\"index\":{\"_index\":\"rollup-docs\",\"_type\":\"doc\"}}\n"); + } String date = String.format(Locale.ROOT, "%04d-01-01T00:%02d:00Z", year, i); bulk.append("{\"timestamp\":\"").append(date).append("\",\"value\":").append(i).append("}\n"); } @@ -257,7 +274,8 @@ public void testRollupAfterRestart() throws Exception { client().performRequest(bulkRequest); // create the rollup job - final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-job-test"); + final Request createRollupJobRequest = new Request("PUT", getRollupEndpoint() + "/job/rollup-job-test"); + createRollupJobRequest.setJsonEntity("{" + "\"index_pattern\":\"rollup-*\"," + "\"rollup_index\":\"results-rollup\"," @@ -278,7 +296,7 @@ public void testRollupAfterRestart() throws Exception { assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE)); // start the rollup job - final Request startRollupJobRequest = new Request("POST", "/_xpack/rollup/job/rollup-job-test/_start"); + final Request startRollupJobRequest = new Request("POST", getRollupEndpoint() + "/job/rollup-job-test/_start"); Map startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest)); assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE)); @@ -304,12 +322,12 @@ public void testRollupIDSchemeAfterRestart() throws Exception { assumeTrue("Rollup ID scheme changed in 6.4", getOldClusterVersion().before(Version.V_6_4_0)); if (isRunningAgainstOldCluster()) { - final Request indexRequest = new Request("POST", "/id-test-rollup/doc/1"); + final Request indexRequest = new Request("POST", "/id-test-rollup" + type + "/1"); indexRequest.setJsonEntity("{\"timestamp\":\"2018-01-01T00:00:01\",\"value\":123}"); client().performRequest(indexRequest); // create the rollup job - final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-id-test"); + final Request createRollupJobRequest = new Request("PUT", getRollupEndpoint() + "/job/rollup-id-test"); createRollupJobRequest.setJsonEntity("{" + "\"index_pattern\":\"id-test-rollup\"," + "\"rollup_index\":\"id-test-results-rollup\"," @@ -337,7 +355,7 @@ public void testRollupIDSchemeAfterRestart() throws Exception { assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE)); // start the rollup job - final Request startRollupJobRequest = new Request("POST", "/_xpack/rollup/job/rollup-id-test/_start"); + final Request startRollupJobRequest = new Request("POST", getRollupEndpoint() + "/job/rollup-id-test/_start"); Map startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest)); assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE)); @@ -365,9 +383,11 @@ public void testRollupIDSchemeAfterRestart() throws Exception { } else { - final Request indexRequest = new Request("POST", "/id-test-rollup/doc/2"); + final Request indexRequest = new Request("POST", "/id-test-rollup/" + type + "/2"); indexRequest.setJsonEntity("{\"timestamp\":\"2018-01-02T00:00:01\",\"value\":345}"); - indexRequest.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + indexRequest.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); + } client().performRequest(indexRequest); assertRollUpJob("rollup-id-test"); @@ -431,12 +451,8 @@ public void testSqlFailsOnIndexWithTwoTypes() throws IOException { client().performRequest(doc2); return; } - final Request sqlRequest; - if (isRunningAgainstOldCluster()) { - sqlRequest = new Request("POST", "/_xpack/sql"); - } else { - sqlRequest = new Request("POST", "/_sql"); - } + final Request sqlRequest = new Request("POST", getSQLEndpoint()); + sqlRequest.setJsonEntity("{\"query\":\"SELECT * FROM testsqlfailsonindexwithtwotypes\"}"); ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(sqlRequest)); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); @@ -458,8 +474,12 @@ private void assertOldTemplatesAreDeleted() throws IOException { private void assertWatchIndexContentsWork() throws Exception { // Fetch a basic watch Request getRequest = new Request("GET", "_watcher/watch/bwc_watch"); - getRequest.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE, - WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE)); + getRequest.setOptions( + expectWarnings( + IndexAction.TYPES_DEPRECATION_MESSAGE + ) + ); + Map bwcWatch = entityAsMap(client().performRequest(getRequest)); logger.error("-----> {}", bwcWatch); @@ -475,8 +495,11 @@ private void assertWatchIndexContentsWork() throws Exception { // Fetch a watch with "fun" throttle periods getRequest = new Request("GET", "_watcher/watch/bwc_throttle_period"); - getRequest.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE, - WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE)); + getRequest.setOptions( + expectWarnings( + IndexAction.TYPES_DEPRECATION_MESSAGE + ) + ); bwcWatch = entityAsMap(client().performRequest(getRequest)); assertThat(bwcWatch.get("found"), equalTo(true)); source = (Map) bwcWatch.get("watch"); @@ -560,7 +583,13 @@ private void waitForHits(String indexName, int expectedHits) throws Exception { try { Map response = entityAsMap(client().performRequest(request)); Map hits = (Map) response.get("hits"); - int total = (int) hits.get("total"); + logger.info("Hits are: {}", hits); + int total; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + total = (int) ((Map) hits.get("total")).get("value"); + } else { + total = (int) hits.get("total"); + } assertThat(total, greaterThanOrEqualTo(expectedHits)); } catch (IOException ioe) { if (ioe instanceof ResponseException) { @@ -580,12 +609,7 @@ static String toStr(Response response) throws IOException { private void createUser(final boolean oldCluster) throws Exception { final String id = oldCluster ? "preupgrade_user" : "postupgrade_user"; - Request request; - if (oldCluster) { - request = new Request("PUT", "/_xpack/security/user/" + id); - } else { - request = new Request("PUT", "/_security/user/" + id); - } + Request request = new Request("PUT", getSecurityEndpoint() + "/user/" + id); request.setJsonEntity( "{\n" + " \"password\" : \"j@rV1s\",\n" + @@ -599,12 +623,7 @@ private void createUser(final boolean oldCluster) throws Exception { private void createRole(final boolean oldCluster) throws Exception { final String id = oldCluster ? "preupgrade_role" : "postupgrade_role"; - Request request; - if (oldCluster) { - request = new Request("PUT", "/_xpack/security/role/" + id); - } else { - request = new Request("PUT", "/_security/role/" + id); - } + Request request = new Request("PUT", getSecurityEndpoint() + "/role/" + id); request.setJsonEntity( "{\n" + " \"run_as\": [ \"abc\" ],\n" + @@ -625,20 +644,59 @@ private void createRole(final boolean oldCluster) throws Exception { private void assertUserInfo(final boolean oldCluster) throws Exception { final String user = oldCluster ? "preupgrade_user" : "postupgrade_user"; - Map response = oldCluster ? - entityAsMap(client().performRequest(new Request("GET", "/_xpack/security/user/" + user))) : - entityAsMap(client().performRequest(new Request("GET", "/_security/user/" + user))); + Request request = new Request("GET", getSecurityEndpoint() + "/user/" + user);; + Map response = entityAsMap(client().performRequest(request)); @SuppressWarnings("unchecked") Map userInfo = (Map) response.get(user); assertEquals(user + "@example.com", userInfo.get("email")); assertNotNull(userInfo.get("full_name")); assertNotNull(userInfo.get("roles")); } + private String getSecurityEndpoint() { + String securityEndpoint; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + securityEndpoint = "/_security"; + } else { + securityEndpoint = "/_xpack/security"; + } + return securityEndpoint; + } + + private String getSQLEndpoint() { + String securityEndpoint; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + securityEndpoint = "/_sql"; + } else { + securityEndpoint = "/_xpack/sql"; + } + return securityEndpoint; + } + + private String getRollupEndpoint() { + String securityEndpoint; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + securityEndpoint = "/_rollup"; + } else { + securityEndpoint = "/_xpack/rollup"; + } + return securityEndpoint; + } + + private String getWatcherEndpoint() { + String securityEndpoint; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + securityEndpoint = "/_watcher"; + } else { + securityEndpoint = "/_xpack/watcher"; + } + return securityEndpoint; + } + private void assertRoleInfo(final boolean oldCluster) throws Exception { final String role = oldCluster ? "preupgrade_role" : "postupgrade_role"; - @SuppressWarnings("unchecked") Map response = oldCluster ? - (Map) entityAsMap(client().performRequest(new Request("GET", "/_xpack/security/role/" + role))).get(role) : - (Map) entityAsMap(client().performRequest(new Request("GET", "/_security/role/" + role))).get(role); + @SuppressWarnings("unchecked") Map response = (Map) entityAsMap( + client().performRequest(new Request("GET", getSecurityEndpoint() + "/role/" + role)) + ).get(role); assertNotNull(response.get("run_as")); assertNotNull(response.get("cluster")); assertNotNull(response.get("indices")); @@ -650,12 +708,7 @@ private void assertRollUpJob(final String rollupJob) throws Exception { waitForRollUpJob(rollupJob, expectedStates); // check that the rollup job is started using the RollUp API - final Request getRollupJobRequest; - if (isRunningAgainstOldCluster()) { - getRollupJobRequest = new Request("GET", "/_xpack/rollup/job/" + rollupJob); - } else { - getRollupJobRequest = new Request("GET", "/_rollup/job/" + rollupJob); - } + final Request getRollupJobRequest = new Request("GET", getRollupEndpoint() + "/job/" + rollupJob); Map getRollupJobResponse = entityAsMap(client().performRequest(getRollupJobRequest)); Map job = getJob(getRollupJobResponse, rollupJob); assertNotNull(job); @@ -700,12 +753,8 @@ private void assertRollUpJob(final String rollupJob) throws Exception { private void waitForRollUpJob(final String rollupJob, final Matcher expectedStates) throws Exception { assertBusy(() -> { - final Request getRollupJobRequest; - if (isRunningAgainstOldCluster()) { - getRollupJobRequest = new Request("GET", "/_xpack/rollup/job/" + rollupJob); - } else { - getRollupJobRequest = new Request("GET", "/_rollup/job/" + rollupJob); - } + final Request getRollupJobRequest = new Request("GET", getRollupEndpoint() + "/job/" + rollupJob); + Response getRollupJobResponse = client().performRequest(getRollupJobRequest); assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); From 757d5b11c91c34ccb39653bde44f8adb97a7d164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 7 Feb 2019 15:56:42 +0100 Subject: [PATCH 30/43] Remove unneeded branches testing 6.0 bwc --- .../elasticsearch/upgrades/IndexingIT.java | 28 ++----------------- .../TokenBackwardsCompatibilityIT.java | 24 ---------------- 2 files changed, 2 insertions(+), 50 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index 8d8d999a55b8d..b44ae0360d884 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -6,20 +6,15 @@ package org.elasticsearch.upgrades; import org.apache.http.util.EntityUtils; -import org.elasticsearch.common.Booleans; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.action.document.RestBulkAction; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.rest.action.document.RestBulkAction; import java.io.IOException; import java.nio.charset.StandardCharsets; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; -import static org.hamcrest.Matchers.equalTo; /** * Basic test that indexed documents survive the rolling restart. @@ -53,25 +48,6 @@ public void testIndexing() throws IOException { } if (CLUSTER_TYPE == ClusterType.OLD) { - { - Version minimumIndexCompatibilityVersion = Version.CURRENT.minimumIndexCompatibilityVersion(); - assertThat("this branch is not needed if we aren't compatible with 6.0", - minimumIndexCompatibilityVersion.onOrBefore(Version.V_6_0_0), equalTo(true)); - if (minimumIndexCompatibilityVersion.before(Version.V_7_0_0)) { - XContentBuilder template = jsonBuilder(); - template.startObject(); - { - template.field("index_patterns", "*"); - template.startObject("settings"); - template.field("number_of_shards", 5); - template.endObject(); - } - template.endObject(); - Request createTemplate = new Request("PUT", "/_template/template"); - createTemplate.setJsonEntity(Strings.toString(template)); - client().performRequest(createTemplate); - } - } Request createTestIndex = new Request("PUT", "/test_index"); createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}"); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index 1f0b8cffe20b0..0b07713c317ce 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -13,8 +13,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.action.document.RestGetAction; import org.elasticsearch.test.rest.yaml.ObjectPath; @@ -23,32 +21,10 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.equalTo; - public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase { public void testGeneratingTokenInOldCluster() throws Exception { assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD); - { - Version minimumIndexCompatibilityVersion = Version.CURRENT.minimumIndexCompatibilityVersion(); - assertThat("this branch is not needed if we aren't compatible with 6.0", - minimumIndexCompatibilityVersion.onOrBefore(Version.V_6_0_0), equalTo(true)); - if (minimumIndexCompatibilityVersion.before(Version.V_7_0_0)) { - XContentBuilder template = jsonBuilder(); - template.startObject(); - { - template.field("index_patterns", "*"); - template.startObject("settings"); - template.field("number_of_shards", 5); - template.endObject(); - } - template.endObject(); - Request createTemplate = new Request("PUT", "/_template/template"); - createTemplate.setJsonEntity(Strings.toString(template)); - client().performRequest(createTemplate); - } - } Request createTokenRequest = new Request("POST", "/_security/oauth2/token"); createTokenRequest.setJsonEntity( From 95c9f2e1c9fe0fa3aede218675f13114f61ffcdc Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 17:43:51 +0200 Subject: [PATCH 31/43] Completly fix x-pack full cluster restart --- .../upgrades/FullClusterRestartIT.java | 4 +- .../xpack/restart/FullClusterRestartIT.java | 39 ++++++++++++++----- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 19c7a1d5ce87b..4c447c6010619 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -908,7 +908,7 @@ public void testSnapshotRestore() throws IOException { // In 7.0, type names are no longer expected by default in put index template requests. // We therefore use the deprecated typed APIs when running against the current version. - if (isRunningAgainstOldCluster() == false && getOldClusterVersion().major < 7) { + if (isRunningAgainstAncientCluster()) { createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); } createTemplateRequest.setOptions(allowTypeRemovalWarnings()); @@ -1212,7 +1212,7 @@ private void saveInfoDocument(String type, String value) throws IOException { private String loadInfoDocument(String type) throws IOException { Request request = new Request("GET", "/info/" + this.type + "/" + index + "_" + type); request.addParameter("filter_path", "_source"); - if (isRunningAgainstAncientCluster() == false && getOldClusterVersion().major < 7) { + if (isRunningAgainstAncientCluster()) { request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); } String doc = toStr(client().performRequest(request)); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 53658eba0cea2..c7bea7b108804 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger; import org.hamcrest.Matcher; @@ -474,11 +475,20 @@ private void assertOldTemplatesAreDeleted() throws IOException { private void assertWatchIndexContentsWork() throws Exception { // Fetch a basic watch Request getRequest = new Request("GET", "_watcher/watch/bwc_watch"); - getRequest.setOptions( - expectWarnings( - IndexAction.TYPES_DEPRECATION_MESSAGE - ) - ); + if (getOldClusterVersion().before(Version.V_7_0_0)) { + getRequest.setOptions( + expectWarnings( + IndexAction.TYPES_DEPRECATION_MESSAGE, + WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE + ) + ); + } else { + getRequest.setOptions( + expectWarnings( + IndexAction.TYPES_DEPRECATION_MESSAGE + ) + ); + } Map bwcWatch = entityAsMap(client().performRequest(getRequest)); @@ -495,11 +505,20 @@ private void assertWatchIndexContentsWork() throws Exception { // Fetch a watch with "fun" throttle periods getRequest = new Request("GET", "_watcher/watch/bwc_throttle_period"); - getRequest.setOptions( - expectWarnings( - IndexAction.TYPES_DEPRECATION_MESSAGE - ) - ); + if (getOldClusterVersion().before(Version.V_7_0_0)) { + getRequest.setOptions( + expectWarnings( + IndexAction.TYPES_DEPRECATION_MESSAGE, + WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE + ) + ); + } else { + getRequest.setOptions( + expectWarnings( + IndexAction.TYPES_DEPRECATION_MESSAGE + ) + ); + } bwcWatch = entityAsMap(client().performRequest(getRequest)); assertThat(bwcWatch.get("found"), equalTo(true)); source = (Map) bwcWatch.get("watch"); From cee84556119072904e92598748ff0eddc59af642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 7 Feb 2019 17:16:37 +0100 Subject: [PATCH 32/43] Use typeless endpoint in TokenBackwardsCompatibilityIT --- .../upgrades/TokenBackwardsCompatibilityIT.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index 0b07713c317ce..ff450ea8c83f0 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -40,7 +40,7 @@ public void testGeneratingTokenInOldCluster() throws Exception { assertNotNull(token); assertTokenWorks(token); - Request indexRequest1 = new Request("PUT", "token_backwards_compatibility_it/doc/old_cluster_token1"); + Request indexRequest1 = new Request("PUT", "token_backwards_compatibility_it/_doc/old_cluster_token1"); indexRequest1.setJsonEntity( "{\n" + " \"token\": \"" + token + "\"\n" + @@ -54,7 +54,7 @@ public void testGeneratingTokenInOldCluster() throws Exception { token = (String) responseMap.get("access_token"); assertNotNull(token); assertTokenWorks(token); - Request indexRequest2 = new Request("PUT", "token_backwards_compatibility_it/doc/old_cluster_token2"); + Request indexRequest2 = new Request("PUT", "token_backwards_compatibility_it/_doc/old_cluster_token2"); indexRequest2.setJsonEntity( "{\n" + " \"token\": \"" + token + "\"\n" + @@ -65,7 +65,7 @@ public void testGeneratingTokenInOldCluster() throws Exception { public void testTokenWorksInMixedOrUpgradedCluster() throws Exception { assumeTrue("this test should only run against the mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.MIXED || CLUSTER_TYPE == ClusterType.UPGRADED); - Request getRequest = new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token1"); + Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token1"); getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Response getResponse = client().performRequest(getRequest); assertOK(getResponse); @@ -76,7 +76,7 @@ public void testTokenWorksInMixedOrUpgradedCluster() throws Exception { public void testMixedCluster() throws Exception { assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); assumeTrue("the master must be on the latest version before we can write", isMasterOnLatestVersion()); - Request getRequest = new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token2"); + Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token2"); getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Response getResponse = client().performRequest(getRequest); @@ -128,7 +128,7 @@ public void testMixedCluster() throws Exception { public void testUpgradedCluster() throws Exception { assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.UPGRADED); - Request getRequest = new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token2"); + Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token2"); getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Response getResponse = client().performRequest(getRequest); @@ -144,7 +144,7 @@ public void testUpgradedCluster() throws Exception { assertOK(invalidationResponse); assertTokenDoesNotWork(token); - getRequest = new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token1"); + getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token1"); getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); getResponse = client().performRequest(getRequest); From 9ce5b19399fb771bc2d5e2d2f37b26ab3bcf5aa8 Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Thu, 7 Feb 2019 18:28:51 +0200 Subject: [PATCH 33/43] Fix watcher tests --- .../elasticsearch/xpack/watcher/WatcherIndexingListener.java | 2 +- .../xpack/watcher/actions/index/IndexActionTests.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java index 48e8dd7813e19..08ab9e9cec42a 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java @@ -102,7 +102,7 @@ void setConfiguration(Configuration configuration) { @Override public Engine.Index preIndex(ShardId shardId, Engine.Index operation) { if (isWatchDocument(shardId.getIndexName(), operation.type())) { - ZonedDateTime now = Instant.ofEpochMilli(clock.millis()).atZone(ZoneOffset.UTC); + ZonedDateTime now = clock.instant().atZone(ZoneOffset.UTC); try { Watch watch = parser.parseWithSecrets(operation.id(), true, operation.source(), now, XContentType.JSON, operation.getIfSeqNo(), operation.getIfPrimaryTerm()); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java index da702e01047bd..57861121c3133 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java @@ -36,6 +36,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -324,8 +325,9 @@ public void testIndexActionExecuteSingleDoc() throws Exception { assertThat(indexRequest.getRefreshPolicy(), is(expectedRefreshPolicy)); if (timestampField != null) { + final DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; assertThat(indexRequest.sourceAsMap().keySet(), is(hasSize(2))); - assertThat(indexRequest.sourceAsMap(), hasEntry(timestampField, executionTime.toString())); + assertThat(indexRequest.sourceAsMap(), hasEntry(timestampField, formatter.format(executionTime))); } else { assertThat(indexRequest.sourceAsMap().keySet(), is(hasSize(1))); } From a542719267dafd99f68edd40ee5d2293ed9cd427 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 7 Feb 2019 18:50:16 +0200 Subject: [PATCH 34/43] Fix checkstyle --- .../org/elasticsearch/xpack/watcher/WatcherIndexingListener.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java index 08ab9e9cec42a..81fda0c0bd47c 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.time.Clock; -import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; From 71c88f92d9cc22990c88e7c705f898b3d6e30d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 7 Feb 2019 17:51:02 +0100 Subject: [PATCH 35/43] Fix old_cluster/40_ml_datafeed_crud.yml Going back from 8.0 we don't parse types here. --- .../rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml index bbe26eb69bd20..d7dba6b3b2d3a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml @@ -28,7 +28,6 @@ { "job_id":"old-cluster-datafeed-job", "indices":["airline-data"], - "types":["response"], "scroll_size": 2000 } From 33a419e66a542dba3e8d98ebdb175909d619ef53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 7 Feb 2019 18:15:21 +0100 Subject: [PATCH 36/43] Fix TokenBackwardsCompatibilityIT Don't expect type deprecation warnings when using type-less get requests. --- .../upgrades/TokenBackwardsCompatibilityIT.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index ff450ea8c83f0..6c5fa0c1a9704 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -13,7 +13,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.rest.action.document.RestGetAction; import org.elasticsearch.test.rest.yaml.ObjectPath; import java.io.IOException; @@ -66,7 +65,6 @@ public void testTokenWorksInMixedOrUpgradedCluster() throws Exception { assumeTrue("this test should only run against the mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.MIXED || CLUSTER_TYPE == ClusterType.UPGRADED); Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token1"); - getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Response getResponse = client().performRequest(getRequest); assertOK(getResponse); Map source = (Map) entityAsMap(getResponse).get("_source"); @@ -77,7 +75,6 @@ public void testMixedCluster() throws Exception { assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); assumeTrue("the master must be on the latest version before we can write", isMasterOnLatestVersion()); Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token2"); - getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Response getResponse = client().performRequest(getRequest); Map source = (Map) entityAsMap(getResponse).get("_source"); @@ -129,7 +126,6 @@ public void testMixedCluster() throws Exception { public void testUpgradedCluster() throws Exception { assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.UPGRADED); Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token2"); - getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); Response getResponse = client().performRequest(getRequest); assertOK(getResponse); @@ -145,7 +141,6 @@ public void testUpgradedCluster() throws Exception { assertTokenDoesNotWork(token); getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token1"); - getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); getResponse = client().performRequest(getRequest); source = (Map) entityAsMap(getResponse).get("_source"); From 0f11822b9a4f54941d34f737b4cfd84b60006943 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 7 Feb 2019 13:07:45 -0700 Subject: [PATCH 37/43] Fix transport test --- .../transport/AbstractSimpleTransportTestCase.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 87343d4c82087..db00ed028d4e2 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -2387,7 +2387,7 @@ public String executor() { assertEquals(1, transportStats.getRxCount()); assertEquals(1, transportStats.getTxCount()); assertEquals(25, transportStats.getRxSize().getBytes()); - assertEquals(50, transportStats.getTxSize().getBytes()); + assertEquals(51, transportStats.getTxSize().getBytes()); }); serviceC.sendRequest(connection, "internal:action", new TestRequest("hello world"), TransportRequestOptions.EMPTY, transportResponseHandler); @@ -2397,7 +2397,7 @@ public String executor() { assertEquals(1, transportStats.getRxCount()); assertEquals(2, transportStats.getTxCount()); assertEquals(25, transportStats.getRxSize().getBytes()); - assertEquals(106, transportStats.getTxSize().getBytes()); + assertEquals(107, transportStats.getTxSize().getBytes()); }); sendResponseLatch.countDown(); responseLatch.await(); @@ -2405,7 +2405,7 @@ public String executor() { assertEquals(2, stats.getRxCount()); assertEquals(2, stats.getTxCount()); assertEquals(46, stats.getRxSize().getBytes()); - assertEquals(106, stats.getTxSize().getBytes()); + assertEquals(107, stats.getTxSize().getBytes()); } finally { serviceC.close(); } @@ -2502,7 +2502,7 @@ public String executor() { assertEquals(1, transportStats.getRxCount()); assertEquals(1, transportStats.getTxCount()); assertEquals(25, transportStats.getRxSize().getBytes()); - assertEquals(50, transportStats.getTxSize().getBytes()); + assertEquals(51, transportStats.getTxSize().getBytes()); }); serviceC.sendRequest(connection, "internal:action", new TestRequest("hello world"), TransportRequestOptions.EMPTY, transportResponseHandler); @@ -2512,7 +2512,7 @@ public String executor() { assertEquals(1, transportStats.getRxCount()); assertEquals(2, transportStats.getTxCount()); assertEquals(25, transportStats.getRxSize().getBytes()); - assertEquals(106, transportStats.getTxSize().getBytes()); + assertEquals(107, transportStats.getTxSize().getBytes()); }); sendResponseLatch.countDown(); responseLatch.await(); @@ -2527,7 +2527,7 @@ public String executor() { // 49 bytes are the non-exception message bytes that have been received. It should include the initial // handshake message and the header, version, etc bytes in the exception message. assertEquals(failedMessage, 49 + streamOutput.bytes().length(), stats.getRxSize().getBytes()); - assertEquals(106, stats.getTxSize().getBytes()); + assertEquals(107, stats.getTxSize().getBytes()); } finally { serviceC.close(); } From ce62c091d61c87af3bb7de9c353121d1e43ac3ce Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 7 Feb 2019 13:15:46 -0800 Subject: [PATCH 38/43] fix checkstyle --- .../upgrades/AbstractFullClusterRestartTestCase.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index 97e8e87e5dc4d..18215b6797ec1 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -20,17 +20,8 @@ package org.elasticsearch.upgrades; import org.elasticsearch.Version; -import org.elasticsearch.client.Request; import org.elasticsearch.common.Booleans; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.rest.ESRestTestCase; -import org.junit.Before; - -import java.io.IOException; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.equalTo; public abstract class AbstractFullClusterRestartTestCase extends ESRestTestCase { From 9990616f4f6176b6a6a411a5b2ad09b89aea859e Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 7 Feb 2019 14:44:52 -0700 Subject: [PATCH 39/43] Add awaits fix --- .../xpack/watcher/actions/index/IndexActionTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java index 57861121c3133..8ce8a15e25e87 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java @@ -276,6 +276,7 @@ public void testConfigureIndexInMapAndAction() { fieldName + "] or [ctx.payload._doc." + fieldName + "]")); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38581") public void testIndexActionExecuteSingleDoc() throws Exception { boolean customId = randomBoolean(); boolean docIdAsParam = customId && randomBoolean(); From 195a4c380b3d33b757d6f3c9bafec2a40bbdd54e Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 7 Feb 2019 19:01:45 -0500 Subject: [PATCH 40/43] Revert errant change to Dockerfile --- distribution/docker/src/docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index c202fa78668f5..37bbfe6688597 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -30,13 +30,13 @@ RUN groupadd -g 1000 elasticsearch && \ WORKDIR /usr/share/elasticsearch -${source_elasticsearch} - +COPY ${elasticsearch} /opt/ RUN tar zxf /opt/${elasticsearch} --strip-components=1 RUN mkdir -p config data logs RUN chmod 0775 config data logs COPY config/elasticsearch.yml config/log4j2.properties config/ + ################################################################################ # Build stage 1 (the actual elasticsearch image): # Copy elasticsearch from stage 0 From ba8ed181001e20e324ea24adff89a608a7c78936 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 7 Feb 2019 17:03:52 -0700 Subject: [PATCH 41/43] Mute --- .../java/org/elasticsearch/index/seqno/RetentionLeaseIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java index 13dadd051c273..a2880c0d330ad 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java @@ -260,6 +260,7 @@ public void testBackgroundRetentionLeaseSync() throws Exception { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38588") public void testRetentionLeasesSyncOnRecovery() throws Exception { final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2); internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas); From 97e49644cae15ce037a8daf22493c89390fda456 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Fri, 8 Feb 2019 14:01:44 +1100 Subject: [PATCH 42/43] There might not be any release compat versions Most recent released version might be before the minimum compat version (released 6.6.0, staging 7.0.0, current 8.0.0) In this case don't worry about "released" checks --- .../plugins/RemovePluginCommandTests.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java index c62d37a4e28d8..415be8f2bbd19 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java @@ -137,12 +137,21 @@ public void testBasic() throws Exception { } public void testRemoveOldVersion() throws Exception { + Version previous = VersionUtils.getPreviousVersion(); + if (previous.before(Version.CURRENT.minimumIndexCompatibilityVersion()) ) { + // Can happen when bumping majors: 8.0 is only compat back to 7.0, but that's not released yet + // In this case, ignore what's released and just find that latest version before current + previous = VersionUtils.allVersions().stream() + .filter(v -> v.before(Version.CURRENT)) + .max(Version::compareTo) + .get(); + } createPlugin( "fake", VersionUtils.randomVersionBetween( random(), Version.CURRENT.minimumIndexCompatibilityVersion(), - VersionUtils.getPreviousVersion())); + previous)); removePlugin("fake", home, randomBoolean()); assertThat(Files.exists(env.pluginsFile().resolve("fake")), equalTo(false)); assertRemoveCleaned(env); From 94fefca6773323dd9244fd14bf146319fe42810f Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Fri, 8 Feb 2019 14:24:10 +1100 Subject: [PATCH 43/43] Only "include_type_name" if running on >= 7 In cluster restart tests, we need to "include_type_name" if the cluster includes a pre-7 version, but the test is running against a 7+ version --- .../java/org/elasticsearch/upgrades/FullClusterRestartIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 4c447c6010619..c6713a2907e33 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -907,8 +907,8 @@ public void testSnapshotRestore() throws IOException { createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder)); // In 7.0, type names are no longer expected by default in put index template requests. - // We therefore use the deprecated typed APIs when running against the current version. - if (isRunningAgainstAncientCluster()) { + // We therefore use the deprecated typed APIs when running against the current version, but testing with a pre-7 version + if (isRunningAgainstOldCluster() == false && getOldClusterVersion().major < 7) { createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); } createTemplateRequest.setOptions(allowTypeRemovalWarnings());