From c3d46152bbd340a09dcb321bf93b37eea5517cfd Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 8 Jan 2020 18:22:37 -0800 Subject: [PATCH 01/29] Voting config exclusions should work with absent nodes --- .../AddVotingConfigExclusionsRequest.java | 18 +++-- .../coordination/CoordinationMetaData.java | 7 +- .../cluster/node/DiscoveryNodes.java | 68 +++++++++++++++---- ...AddVotingConfigExclusionsRequestTests.java | 4 -- ...tAddVotingConfigExclusionsActionTests.java | 33 ++------- 5 files changed, 78 insertions(+), 52 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index df11435aba1c9..e3d236a953545 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -27,6 +27,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.iterable.Iterables; +import org.elasticsearch.common.util.set.Sets; import java.io.IOException; import java.util.Arrays; @@ -72,16 +74,22 @@ public AddVotingConfigExclusionsRequest(StreamInput in) throws IOException { Set resolveVotingConfigExclusions(ClusterState currentState) { final DiscoveryNodes allNodes = currentState.nodes(); - final Set resolvedNodes = Arrays.stream(allNodes.resolveNodes(nodeDescriptions)) + final DiscoveryNodes.NodeResolutionResults nodeResolutionResults = allNodes.resolveNodes(nodeDescriptions, true); + + final Set resolvedNodes = Arrays.stream(nodeResolutionResults.getResolvedNodes()) .map(allNodes::get).filter(DiscoveryNode::isMasterNode).map(VotingConfigExclusion::new).collect(Collectors.toSet()); + final Set unresolvedNodes = Arrays.stream(nodeResolutionResults.getUnresolvedNodes()) + .map(VotingConfigExclusion::new).collect(Collectors.toSet()); + + Set allProcessedNodes = Sets.newHashSet(Iterables.concat(resolvedNodes, unresolvedNodes)); - if (resolvedNodes.isEmpty()) { + if (allProcessedNodes.isEmpty()) { throw new IllegalArgumentException("add voting config exclusions request for " + Arrays.asList(nodeDescriptions) - + " matched no master-eligible nodes"); + + " matched no master-eligible nodes or absent nodes"); } - resolvedNodes.removeIf(n -> currentState.getVotingConfigExclusions().contains(n)); - return resolvedNodes; + allProcessedNodes.removeIf(n -> currentState.getVotingConfigExclusions().contains(n)); + return allProcessedNodes; } Set resolveVotingConfigExclusionsAndCheckMaximum(ClusterState currentState, int maxExclusionsCount, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java index cfd4456062e35..38a44502b5937 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java @@ -190,7 +190,7 @@ public static class Builder { public Builder() { } - + public Builder(CoordinationMetaData state) { this.term = state.term; this.lastCommittedConfiguration = state.lastCommittedConfiguration; @@ -246,6 +246,11 @@ public VotingConfigExclusion(String nodeId, String nodeName) { this.nodeName = nodeName; } + public VotingConfigExclusion(String nodeIdAndName) { + this.nodeId = nodeIdAndName; + this.nodeName = nodeIdAndName; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(nodeId); diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index fd4e7023cfee9..efd73e41e9e87 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -317,6 +317,24 @@ public DiscoveryNode resolveNode(String node) { return nodes.get(resolvedNodeIds[0]); } + public static class NodeResolutionResults { + private String[] resolvedNodes; + private String[] unresolvedNodes; + + public NodeResolutionResults(String[] resolvedNodes, String[] unresolvedNodes) { + this.resolvedNodes = resolvedNodes; + this.unresolvedNodes = unresolvedNodes; + } + + public String[] getResolvedNodes() { + return resolvedNodes; + } + + public String[] getUnresolvedNodes() { + return unresolvedNodes; + } + } + /** * resolves a set of node "descriptions" to concrete and existing node ids. "descriptions" can be (resolved in this order): * - "_local" or "_master" for the relevant nodes @@ -326,60 +344,76 @@ public DiscoveryNode resolveNode(String node) { * or a generic node attribute name in which case value will be treated as a wildcard and matched against the node attribute values. */ public String[] resolveNodes(String... nodes) { + return resolveNodes(nodes, false).getResolvedNodes(); + } + + public NodeResolutionResults resolveNodes(String[] nodes, boolean returnUnresolved) { if (nodes == null || nodes.length == 0) { - return StreamSupport.stream(this.spliterator(), false).map(DiscoveryNode::getId).toArray(String[]::new); + return new NodeResolutionResults(StreamSupport.stream(this.spliterator(), false) + .map(DiscoveryNode::getId).toArray(String[]::new), new String[0]); } else { ObjectHashSet resolvedNodesIds = new ObjectHashSet<>(nodes.length); - for (String nodeId : nodes) { - if (nodeId.equals("_local")) { + ObjectHashSet unresolvedNodesIds = new ObjectHashSet<>(); + boolean nodeIdResolved = false; + + for (String nodeIdToBeProcessed : nodes) { + if (nodeIdToBeProcessed.equals("_local")) { String localNodeId = getLocalNodeId(); if (localNodeId != null) { resolvedNodesIds.add(localNodeId); + nodeIdResolved = true; } - } else if (nodeId.equals("_master")) { + } else if (nodeIdToBeProcessed.equals("_master")) { String masterNodeId = getMasterNodeId(); if (masterNodeId != null) { resolvedNodesIds.add(masterNodeId); + nodeIdResolved = true; } - } else if (nodeExists(nodeId)) { - resolvedNodesIds.add(nodeId); + } else if (nodeExists(nodeIdToBeProcessed)) { + resolvedNodesIds.add(nodeIdToBeProcessed); + nodeIdResolved = true; } else { for (DiscoveryNode node : this) { - if ("_all".equals(nodeId) - || Regex.simpleMatch(nodeId, node.getName()) - || Regex.simpleMatch(nodeId, node.getHostAddress()) - || Regex.simpleMatch(nodeId, node.getHostName())) { + if ("_all".equals(nodeIdToBeProcessed) + || Regex.simpleMatch(nodeIdToBeProcessed, node.getName()) + || Regex.simpleMatch(nodeIdToBeProcessed, node.getHostAddress()) + || Regex.simpleMatch(nodeIdToBeProcessed, node.getHostName())) { resolvedNodesIds.add(node.getId()); + nodeIdResolved = true; } } - int index = nodeId.indexOf(':'); + int index = nodeIdToBeProcessed.indexOf(':'); if (index != -1) { - String matchAttrName = nodeId.substring(0, index); - String matchAttrValue = nodeId.substring(index + 1); + String matchAttrName = nodeIdToBeProcessed.substring(0, index); + String matchAttrValue = nodeIdToBeProcessed.substring(index + 1); if (DiscoveryNodeRole.DATA_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(dataNodes.keys()); } else { resolvedNodesIds.removeAll(dataNodes.keys()); } + nodeIdResolved = true; } else if (DiscoveryNodeRole.MASTER_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(masterNodes.keys()); } else { resolvedNodesIds.removeAll(masterNodes.keys()); } + nodeIdResolved = true; } else if (DiscoveryNodeRole.INGEST_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(ingestNodes.keys()); } else { resolvedNodesIds.removeAll(ingestNodes.keys()); } + nodeIdResolved = true; } else if (DiscoveryNode.COORDINATING_ONLY.equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(getCoordinatingOnlyNodes().keys()); } else { resolvedNodesIds.removeAll(getCoordinatingOnlyNodes().keys()); } + nodeIdResolved = true; } else { for (DiscoveryNode node : this) { for (DiscoveryNodeRole role : Sets.difference(node.getRoles(), DiscoveryNodeRole.BUILT_IN_ROLES)) { @@ -389,6 +423,7 @@ public String[] resolveNodes(String... nodes) { } else { resolvedNodesIds.remove(node.getId()); } + nodeIdResolved = true; } } } @@ -398,14 +433,19 @@ public String[] resolveNodes(String... nodes) { String attrValue = entry.getValue(); if (Regex.simpleMatch(matchAttrName, attrName) && Regex.simpleMatch(matchAttrValue, attrValue)) { resolvedNodesIds.add(node.getId()); + nodeIdResolved = true; } } } } } } + + if(!nodeIdResolved) { + unresolvedNodesIds.add(nodeIdToBeProcessed); + } } - return resolvedNodesIds.toArray(String.class); + return new NodeResolutionResults(resolvedNodesIds.toArray(String.class), unresolvedNodesIds.toArray(String.class)); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index 44880d5bdde59..03ca8652e71c3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -93,10 +93,6 @@ public void testResolve() { contains(localNodeExclusion)); assertThat(makeRequest("other*").resolveVotingConfigExclusions(clusterState), containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); - - assertThat(expectThrows(IllegalArgumentException.class, - () -> makeRequest("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), - equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); } public void testResolveAndCheckMaximum() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index d4dda501f5395..6139e54b155e3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -243,42 +243,19 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc contains(otherNode1Exclusion)); } - public void testReturnsErrorIfNoMatchingNodes() throws InterruptedException { + public void testMatchesAbsentNodes() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); - final SetOnce exceptionHolder = new SetOnce<>(); - - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"not-a-node"}), - expectError(e -> { - exceptionHolder.set(e); - countDownLatch.countDown(); - }) - ); - - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - final Throwable rootCause = exceptionHolder.get().getRootCause(); - assertThat(rootCause, instanceOf(IllegalArgumentException.class)); - assertThat(rootCause.getMessage(), - equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); - } - - public void testOnlyMatchesMasterEligibleNodes() throws InterruptedException { - final CountDownLatch countDownLatch = new CountDownLatch(1); - final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_all", "master:false"}), - expectError(e -> { - exceptionHolder.set(e); + new AddVotingConfigExclusionsRequest(new String[]{"absent_node"}), + expectSuccess(e -> { countDownLatch.countDown(); }) ); assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - final Throwable rootCause = exceptionHolder.get().getRootCause(); - assertThat(rootCause, instanceOf(IllegalArgumentException.class)); - assertThat(rootCause.getMessage(), - equalTo("add voting config exclusions request for [_all, master:false] matched no master-eligible nodes")); + assertEquals(Set.of(new VotingConfigExclusion("absent_node")), + clusterService.getClusterApplierService().state().getVotingConfigExclusions()); } public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedException { From 39eb2a11bb95b50a18fb408a52a925f9161275c8 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 16 Jan 2020 23:28:22 -0800 Subject: [PATCH 02/29] Add new APIs to add voting config exclusion just based on node id or name --- .../AddVotingConfigExclusionsRequest.java | 4 +- .../coordination/CoordinationMetaData.java | 8 +- .../cluster/node/DiscoveryNodes.java | 80 +++++++++++-------- .../RestAddVotingConfigExclusionAction.java | 19 ++++- ...AddVotingConfigExclusionsRequestTests.java | 9 --- ...tAddVotingConfigExclusionsActionTests.java | 54 +------------ ...stAddVotingConfigExclusionActionTests.java | 15 ++++ 7 files changed, 82 insertions(+), 107 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index e3d236a953545..3b9e04ade7877 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -74,12 +74,12 @@ public AddVotingConfigExclusionsRequest(StreamInput in) throws IOException { Set resolveVotingConfigExclusions(ClusterState currentState) { final DiscoveryNodes allNodes = currentState.nodes(); - final DiscoveryNodes.NodeResolutionResults nodeResolutionResults = allNodes.resolveNodes(nodeDescriptions, true); + final DiscoveryNodes.NodeResolutionResults nodeResolutionResults = allNodes.resolveNodesExact(nodeDescriptions); final Set resolvedNodes = Arrays.stream(nodeResolutionResults.getResolvedNodes()) .map(allNodes::get).filter(DiscoveryNode::isMasterNode).map(VotingConfigExclusion::new).collect(Collectors.toSet()); final Set unresolvedNodes = Arrays.stream(nodeResolutionResults.getUnresolvedNodes()) - .map(VotingConfigExclusion::new).collect(Collectors.toSet()); + .map(nodeIdOrName -> new VotingConfigExclusion(nodeIdOrName, nodeIdOrName)).collect(Collectors.toSet()); Set allProcessedNodes = Sets.newHashSet(Iterables.concat(resolvedNodes, unresolvedNodes)); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java index 38a44502b5937..340bfba003f6d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java @@ -233,7 +233,8 @@ public static class VotingConfigExclusion implements Writeable, ToXContentFragme private final String nodeName; public VotingConfigExclusion(DiscoveryNode node) { - this(node.getId(), node.getName()); + this.nodeId = node.getId(); + this.nodeName = node.getName(); } public VotingConfigExclusion(StreamInput in) throws IOException { @@ -246,11 +247,6 @@ public VotingConfigExclusion(String nodeId, String nodeName) { this.nodeName = nodeName; } - public VotingConfigExclusion(String nodeIdAndName) { - this.nodeId = nodeIdAndName; - this.nodeName = nodeIdAndName; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(nodeId); diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index efd73e41e9e87..446f7912b2e49 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -335,6 +336,36 @@ public String[] getUnresolvedNodes() { } } + public NodeResolutionResults resolveNodesExact(String... nodes) { + if (nodes == null || nodes.length == 0) { + return new NodeResolutionResults(StreamSupport.stream(this.spliterator(), false) + .map(DiscoveryNode::getId).toArray(String[]::new), new String[0]); + } else { + ObjectHashSet resolvedNodesIds = new ObjectHashSet<>(nodes.length); + ObjectHashSet unresolvedNodesIds = new ObjectHashSet<>(); + + Map existingNodesNameId = new HashMap<>(); + for (DiscoveryNode node : this) { + existingNodesNameId.put(node.getName(), node.getId()); + } + + for (String nodeToBeProcessed : nodes) { + if (nodeExists(nodeToBeProcessed)) { + resolvedNodesIds.add(nodeToBeProcessed); + } + else if (existingNodesNameId.containsKey(nodeToBeProcessed)){ + resolvedNodesIds.add(existingNodesNameId.get(nodeToBeProcessed)); + } + else { + unresolvedNodesIds.add(nodeToBeProcessed); + } + } + + return new NodeResolutionResults(resolvedNodesIds.toArray(String.class), unresolvedNodesIds.toArray(String.class)); + } + } + + /** * resolves a set of node "descriptions" to concrete and existing node ids. "descriptions" can be (resolved in this order): * - "_local" or "_master" for the relevant nodes @@ -344,76 +375,61 @@ public String[] getUnresolvedNodes() { * or a generic node attribute name in which case value will be treated as a wildcard and matched against the node attribute values. */ public String[] resolveNodes(String... nodes) { - return resolveNodes(nodes, false).getResolvedNodes(); - } - - public NodeResolutionResults resolveNodes(String[] nodes, boolean returnUnresolved) { if (nodes == null || nodes.length == 0) { - return new NodeResolutionResults(StreamSupport.stream(this.spliterator(), false) - .map(DiscoveryNode::getId).toArray(String[]::new), new String[0]); + return StreamSupport.stream(this.spliterator(), false).map(DiscoveryNode::getId).toArray(String[]::new); } else { ObjectHashSet resolvedNodesIds = new ObjectHashSet<>(nodes.length); - ObjectHashSet unresolvedNodesIds = new ObjectHashSet<>(); - boolean nodeIdResolved = false; - for (String nodeIdToBeProcessed : nodes) { - if (nodeIdToBeProcessed.equals("_local")) { + for (String nodeId : nodes) { + if (nodeId.equals("_local")) { String localNodeId = getLocalNodeId(); if (localNodeId != null) { resolvedNodesIds.add(localNodeId); - nodeIdResolved = true; } - } else if (nodeIdToBeProcessed.equals("_master")) { + } else if (nodeId.equals("_master")) { String masterNodeId = getMasterNodeId(); if (masterNodeId != null) { resolvedNodesIds.add(masterNodeId); - nodeIdResolved = true; } - } else if (nodeExists(nodeIdToBeProcessed)) { - resolvedNodesIds.add(nodeIdToBeProcessed); - nodeIdResolved = true; + } else if (nodeExists(nodeId)) { + resolvedNodesIds.add(nodeId); } else { for (DiscoveryNode node : this) { - if ("_all".equals(nodeIdToBeProcessed) - || Regex.simpleMatch(nodeIdToBeProcessed, node.getName()) - || Regex.simpleMatch(nodeIdToBeProcessed, node.getHostAddress()) - || Regex.simpleMatch(nodeIdToBeProcessed, node.getHostName())) { + if ("_all".equals(nodeId) + || Regex.simpleMatch(nodeId, node.getName()) + || Regex.simpleMatch(nodeId, node.getHostAddress()) + || Regex.simpleMatch(nodeId, node.getHostName())) { resolvedNodesIds.add(node.getId()); - nodeIdResolved = true; } } - int index = nodeIdToBeProcessed.indexOf(':'); + int index = nodeId.indexOf(':'); if (index != -1) { - String matchAttrName = nodeIdToBeProcessed.substring(0, index); - String matchAttrValue = nodeIdToBeProcessed.substring(index + 1); + String matchAttrName = nodeId.substring(0, index); + String matchAttrValue = nodeId.substring(index + 1); if (DiscoveryNodeRole.DATA_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(dataNodes.keys()); } else { resolvedNodesIds.removeAll(dataNodes.keys()); } - nodeIdResolved = true; } else if (DiscoveryNodeRole.MASTER_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(masterNodes.keys()); } else { resolvedNodesIds.removeAll(masterNodes.keys()); } - nodeIdResolved = true; } else if (DiscoveryNodeRole.INGEST_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(ingestNodes.keys()); } else { resolvedNodesIds.removeAll(ingestNodes.keys()); } - nodeIdResolved = true; } else if (DiscoveryNode.COORDINATING_ONLY.equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(getCoordinatingOnlyNodes().keys()); } else { resolvedNodesIds.removeAll(getCoordinatingOnlyNodes().keys()); } - nodeIdResolved = true; } else { for (DiscoveryNode node : this) { for (DiscoveryNodeRole role : Sets.difference(node.getRoles(), DiscoveryNodeRole.BUILT_IN_ROLES)) { @@ -423,7 +439,6 @@ public NodeResolutionResults resolveNodes(String[] nodes, boolean returnUnresolv } else { resolvedNodesIds.remove(node.getId()); } - nodeIdResolved = true; } } } @@ -433,19 +448,14 @@ public NodeResolutionResults resolveNodes(String[] nodes, boolean returnUnresolv String attrValue = entry.getValue(); if (Regex.simpleMatch(matchAttrName, attrName) && Regex.simpleMatch(matchAttrValue, attrValue)) { resolvedNodesIds.add(node.getId()); - nodeIdResolved = true; } } } } } } - - if(!nodeIdResolved) { - unresolvedNodesIds.add(nodeIdToBeProcessed); - } } - return new NodeResolutionResults(resolvedNodesIds.toArray(String.class), unresolvedNodesIds.toArray(String.class)); + return resolvedNodesIds.toArray(String.class); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java index 018392bb05ad8..8669ff626ae3b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java @@ -36,7 +36,11 @@ public class RestAddVotingConfigExclusionAction extends BaseRestHandler { private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(30L); public RestAddVotingConfigExclusionAction(RestController controller) { + // TODO This API is being deprecated. controller.registerHandler(RestRequest.Method.POST, "/_cluster/voting_config_exclusions/{node_name}", this); + + controller.registerHandler(RestRequest.Method.POST, + "/_cluster/voting_config_exclusions/node_ids_or_names/{node_id_or_names}", this); } @Override @@ -55,9 +59,20 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No } AddVotingConfigExclusionsRequest resolveVotingConfigExclusionsRequest(final RestRequest request) { - String nodeName = request.param("node_name"); + String nodeDescriptions; + + // TODO This request param is being deprecated + if (request.hasParam("node_name")) { + nodeDescriptions = request.param("node_name"); + } + else { + nodeDescriptions = request.param("node_ids_or_names"); + } + + assert !Strings.isNullOrEmpty(nodeDescriptions); + return new AddVotingConfigExclusionsRequest( - Strings.splitStringByCommaToArray(nodeName), + Strings.splitStringByCommaToArray(nodeDescriptions), TimeValue.parseTimeValue(request.param("timeout"), DEFAULT_TIMEOUT, getClass().getSimpleName() + ".timeout") ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index 03ca8652e71c3..dc83d3facdf38 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -35,7 +35,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -87,12 +86,6 @@ public void testResolve() { assertThat(makeRequest().resolveVotingConfigExclusions(clusterState), containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); - assertThat(makeRequest("_all").resolveVotingConfigExclusions(clusterState), - containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); - assertThat(makeRequest("_local").resolveVotingConfigExclusions(clusterState), - contains(localNodeExclusion)); - assertThat(makeRequest("other*").resolveVotingConfigExclusions(clusterState), - containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); } public void testResolveAndCheckMaximum() { @@ -129,8 +122,6 @@ public void testResolveAndCheckMaximum() { assertThat(makeRequest().resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 3, "setting.name"), containsInAnyOrder(localNodeExclusion, otherNode2Exclusion)); - assertThat(makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), - contains(localNodeExclusion)); assertThat(expectThrows(IllegalArgumentException.class, () -> makeRequest().resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name")).getMessage(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 6139e54b155e3..5cb6103cdcfd1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -166,58 +166,6 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); } - - public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedException { - final CountDownLatch countDownLatch = new CountDownLatch(1); - - clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other*"}), - expectSuccess(r -> { - assertNotNull(r); - countDownLatch.countDown(); - }) - ); - - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), - containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); - } - - public void testWithdrawsVotesFromAllMasterEligibleNodes() throws InterruptedException { - final CountDownLatch countDownLatch = new CountDownLatch(1); - - clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_all"}), - expectSuccess(r -> { - assertNotNull(r); - countDownLatch.countDown(); - }) - ); - - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), - containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); - } - - public void testWithdrawsVoteFromLocalNode() throws InterruptedException { - final CountDownLatch countDownLatch = new CountDownLatch(1); - - clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_local"}), - expectSuccess(r -> { - assertNotNull(r); - countDownLatch.countDown(); - }) - ); - - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), - contains(localNodeExclusion)); - } - public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedException { final ClusterState state = clusterService.state(); setState(clusterService, builder(state) @@ -254,7 +202,7 @@ public void testMatchesAbsentNodes() throws InterruptedException { ); assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - assertEquals(Set.of(new VotingConfigExclusion("absent_node")), + assertEquals(Set.of(new VotingConfigExclusion("absent_node", "absent_node")), clusterService.getClusterApplierService().state().getVotingConfigExclusions()); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java index c686497922346..d213701d7e5e0 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java @@ -50,4 +50,19 @@ public void testResolveVotingConfigExclusionsRequest() { String[] expected = {"node-1","node-2", "node-3"}; assertArrayEquals(expected, addVotingConfigExclusionsRequest.getNodeDescriptions()); } + + public void testResolveVotingConfigExclusionsRequestNodeIdsOrNames() { + Map params = new HashMap<>(); + params.put("node_ids_or_names", "node-1,node-2,node-3"); + RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(RestRequest.Method.PUT) + .withPath("/_cluster/voting_config_exclusions/node_ids_or_names") + .withParams(params) + .build(); + + AddVotingConfigExclusionsRequest addVotingConfigExclusionsRequest = action.resolveVotingConfigExclusionsRequest(deprecatedRequest); + String[] expected = {"node-1","node-2", "node-3"}; + assertArrayEquals(expected, addVotingConfigExclusionsRequest.getNodeDescriptions()); + } + } From c3eb5a1f2ed01e8687cbab6b84d443dc037eb66e Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 27 Feb 2020 22:18:22 -0800 Subject: [PATCH 03/29] Address feedback comment --- .../AddVotingConfigExclusionsRequest.java | 85 ++++++++++++++++--- .../cluster/node/DiscoveryNodes.java | 41 +++++---- .../RestAddVotingConfigExclusionAction.java | 54 ++++++++++-- ...AddVotingConfigExclusionsRequestTests.java | 4 +- ...tAddVotingConfigExclusionsActionTests.java | 11 +-- ...stAddVotingConfigExclusionActionTests.java | 35 ++++++-- 6 files changed, 181 insertions(+), 49 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 3b9e04ade7877..759c920bbd7e8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -18,12 +18,14 @@ */ package org.elasticsearch.action.admin.cluster.configuration; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; @@ -41,6 +43,8 @@ */ public class AddVotingConfigExclusionsRequest extends MasterNodeRequest { private final String[] nodeDescriptions; + private String[] nodeIds = null; + private String[] nodeNames = null; private final TimeValue timeout; /** @@ -49,7 +53,7 @@ public class AddVotingConfigExclusionsRequest extends MasterNodeRequest resolveVotingConfigExclusions(ClusterState currentState) { final DiscoveryNodes allNodes = currentState.nodes(); - final DiscoveryNodes.NodeResolutionResults nodeResolutionResults = allNodes.resolveNodesExact(nodeDescriptions); + Set allProcessedNodes = null; + + if (nodeDescriptions.length >= 1) { + allProcessedNodes = Arrays.stream(allNodes.resolveNodes(nodeDescriptions)).map(allNodes::get) + .filter(DiscoveryNode::isMasterNode).map(VotingConfigExclusion::new).collect(Collectors.toSet()); + + if (allProcessedNodes.isEmpty()) { + throw new IllegalArgumentException("add voting config exclusions request for " + Arrays.asList(nodeDescriptions) + + " matched no master-eligible nodes"); + } + } + else { + DiscoveryNodes.NodeResolutionResults nodeResolutionResults; + Set resolvedNodes; + Set unresolvedNodes; + + if (nodeIds.length >= 1) { + nodeResolutionResults = allNodes.resolveNodesExact(true, nodeIds); - final Set resolvedNodes = Arrays.stream(nodeResolutionResults.getResolvedNodes()) + unresolvedNodes = Arrays.stream(nodeResolutionResults.getUnresolvedNodes()) + .map(nodeId -> new VotingConfigExclusion(nodeId, "")).collect(Collectors.toSet()); + } + else { + nodeResolutionResults = allNodes.resolveNodesExact(false, nodeNames); + + unresolvedNodes = Arrays.stream(nodeResolutionResults.getUnresolvedNodes()) + .map(nodeName -> new VotingConfigExclusion("", nodeName)).collect(Collectors.toSet()); + } + + resolvedNodes = Arrays.stream(nodeResolutionResults.getResolvedNodes()) .map(allNodes::get).filter(DiscoveryNode::isMasterNode).map(VotingConfigExclusion::new).collect(Collectors.toSet()); - final Set unresolvedNodes = Arrays.stream(nodeResolutionResults.getUnresolvedNodes()) - .map(nodeIdOrName -> new VotingConfigExclusion(nodeIdOrName, nodeIdOrName)).collect(Collectors.toSet()); - Set allProcessedNodes = Sets.newHashSet(Iterables.concat(resolvedNodes, unresolvedNodes)); + allProcessedNodes = Sets.newHashSet(Iterables.concat(resolvedNodes, unresolvedNodes)); - if (allProcessedNodes.isEmpty()) { - throw new IllegalArgumentException("add voting config exclusions request for " + Arrays.asList(nodeDescriptions) - + " matched no master-eligible nodes or absent nodes"); + if (allProcessedNodes.isEmpty()) { + throw new IllegalArgumentException("add voting config exclusions request for nodeIds " + Arrays.asList(nodeIds) + + " or nodeNames " + Arrays.asList(nodeNames) + " matched no master-eligible nodes or absent nodes"); + } } allProcessedNodes.removeIf(n -> currentState.getVotingConfigExclusions().contains(n)); @@ -114,6 +152,20 @@ public String[] getNodeDescriptions() { return nodeDescriptions; } + /** + * @return ids of the nodes for whom to add voting config exclusions. + */ + public String[] getNodeIds() { + return nodeIds; + } + + /** + * @return names of the nodes for whom to add voting config exclusions. + */ + public String[] getNodeNames() { + return nodeNames; + } + /** * @return how long to wait after adding the exclusions for the nodes to be removed from the voting configuration. */ @@ -129,15 +181,24 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + // TODO should this be removed in the latest version where nodeIds and nodeNames are used? out.writeStringArray(nodeDescriptions); + // TODO which version to use here? + if (out.getVersion() == Version.V_EMPTY) { + out.writeStringArray(nodeIds); + out.writeStringArray(nodeNames); + + } out.writeTimeValue(timeout); } @Override public String toString() { return "AddVotingConfigExclusionsRequest{" + - "nodeDescriptions=" + Arrays.asList(nodeDescriptions) + - ", timeout=" + timeout + + "nodeDescriptions=" + Arrays.asList(nodeDescriptions) + ", " + + "nodeIds=" + Arrays.asList(nodeIds) + ", " + + "nodeNames=" + Arrays.asList(nodeNames) + ", " + + "timeout=" + timeout + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 446f7912b2e49..8f59b1b965c84 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -336,32 +336,41 @@ public String[] getUnresolvedNodes() { } } - public NodeResolutionResults resolveNodesExact(String... nodes) { + public NodeResolutionResults resolveNodesExact(boolean isNodeIds, String... nodes) { if (nodes == null || nodes.length == 0) { return new NodeResolutionResults(StreamSupport.stream(this.spliterator(), false) .map(DiscoveryNode::getId).toArray(String[]::new), new String[0]); } else { - ObjectHashSet resolvedNodesIds = new ObjectHashSet<>(nodes.length); - ObjectHashSet unresolvedNodesIds = new ObjectHashSet<>(); - - Map existingNodesNameId = new HashMap<>(); - for (DiscoveryNode node : this) { - existingNodesNameId.put(node.getName(), node.getId()); - } + ObjectHashSet resolvedNodes = new ObjectHashSet<>(nodes.length); + ObjectHashSet unresolvedNodes = new ObjectHashSet<>(); - for (String nodeToBeProcessed : nodes) { - if (nodeExists(nodeToBeProcessed)) { - resolvedNodesIds.add(nodeToBeProcessed); + if (isNodeIds) { + for (String nodeId : nodes) { + if (nodeExists(nodeId)) { + resolvedNodes.add(nodeId); + } + else { + unresolvedNodes.add(nodeId); + } } - else if (existingNodesNameId.containsKey(nodeToBeProcessed)){ - resolvedNodesIds.add(existingNodesNameId.get(nodeToBeProcessed)); + } + else { + Map existingNodesNameId = new HashMap<>(); + for (DiscoveryNode node : this) { + existingNodesNameId.put(node.getName(), node.getId()); } - else { - unresolvedNodesIds.add(nodeToBeProcessed); + + for (String nodeName : nodes) { + if (existingNodesNameId.containsKey(nodeName)){ + resolvedNodes.add(existingNodesNameId.get(nodeName)); + } + else { + unresolvedNodes.add(nodeName); + } } } - return new NodeResolutionResults(resolvedNodesIds.toArray(String.class), unresolvedNodesIds.toArray(String.class)); + return new NodeResolutionResults(resolvedNodes.toArray(String.class), unresolvedNodes.toArray(String.class)); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java index 8669ff626ae3b..075dab3a64c5f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java @@ -23,24 +23,28 @@ import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.IOException; public class RestAddVotingConfigExclusionAction extends BaseRestHandler { private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(30L); + private static final Logger logger = LogManager.getLogger(RestAddVotingConfigExclusionAction.class); + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); public RestAddVotingConfigExclusionAction(RestController controller) { // TODO This API is being deprecated. controller.registerHandler(RestRequest.Method.POST, "/_cluster/voting_config_exclusions/{node_name}", this); - controller.registerHandler(RestRequest.Method.POST, - "/_cluster/voting_config_exclusions/node_ids_or_names/{node_id_or_names}", this); + controller.registerHandler(RestRequest.Method.POST, "/_cluster/voting_config_exclusions", this); } @Override @@ -59,21 +63,53 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No } AddVotingConfigExclusionsRequest resolveVotingConfigExclusionsRequest(final RestRequest request) { - String nodeDescriptions; + String deprecatedNodeDescription = null; + String nodeIds = null; + String nodeNames = null; - // TODO This request param is being deprecated if (request.hasParam("node_name")) { - nodeDescriptions = request.param("node_name"); + DEPRECATION_LOGGER.deprecatedAndMaybeLog("add_voting_config_exclusion", + "Using [node_name] for adding voting config exclustion will be removed in a future version. " + + "Please use [node_ids] or [node_names] instead"); + deprecatedNodeDescription = request.param("node_name"); } - else { - nodeDescriptions = request.param("node_ids_or_names"); + + if (request.hasParam("node_ids")){ + nodeIds = request.param("node_ids"); } - assert !Strings.isNullOrEmpty(nodeDescriptions); + if (request.hasParam("node_names")){ + nodeNames = request.param("node_names"); + } + + if(!oneAndonlyOneIsSet(deprecatedNodeDescription, nodeIds, nodeNames)) { + throw new IllegalArgumentException("Please set node identifiers correctly. " + + "One and only one of [node_name], [node_names] and [node_ids] has to be set"); + } return new AddVotingConfigExclusionsRequest( - Strings.splitStringByCommaToArray(nodeDescriptions), + Strings.splitStringByCommaToArray(deprecatedNodeDescription), + Strings.splitStringByCommaToArray(nodeIds), + Strings.splitStringByCommaToArray(nodeNames), TimeValue.parseTimeValue(request.param("timeout"), DEFAULT_TIMEOUT, getClass().getSimpleName() + ".timeout") ); } + + private boolean oneAndonlyOneIsSet(String deprecatedNodeDescription, String nodeIds, String nodeNames) { + if(Strings.hasText(deprecatedNodeDescription)) { + return Strings.isNullOrEmpty(nodeIds) && Strings.isNullOrEmpty(nodeNames); + } + else if (Strings.hasText(nodeIds)) { + return Strings.isNullOrEmpty(nodeNames); + } + else if (Strings.hasText(nodeNames)) { + return true; + } + else { + // none of the node identifiers are set + return false; + } + + } + } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index dc83d3facdf38..dfca53764e226 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes.Builder; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -46,7 +47,8 @@ public void testSerialization() throws IOException { descriptions[i] = randomAlphaOfLength(10); } TimeValue timeout = TimeValue.timeValueMillis(between(0, 30000)); - final AddVotingConfigExclusionsRequest originalRequest = new AddVotingConfigExclusionsRequest(descriptions, timeout); + final AddVotingConfigExclusionsRequest originalRequest = new AddVotingConfigExclusionsRequest(descriptions, Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, timeout); final AddVotingConfigExclusionsRequest deserialized = copyWriteable(originalRequest, writableRegistry(), AddVotingConfigExclusionsRequest::new); assertThat(deserialized.getNodeDescriptions(), equalTo(originalRequest.getNodeDescriptions())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 5cb6103cdcfd1..107fd6e75d3a3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes.Builder; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -179,7 +180,7 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc // no observer to reconfigure transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}, TimeValue.ZERO), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}, null, null, TimeValue.ZERO), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -191,18 +192,18 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc contains(otherNode1Exclusion)); } - public void testMatchesAbsentNodes() throws InterruptedException { + public void testMatchesAbsentNodesByNodeNames() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"absent_node"}), + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, new String[]{"absent_node"}, TimeValue.ZERO), expectSuccess(e -> { countDownLatch.countDown(); }) ); assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - assertEquals(Set.of(new VotingConfigExclusion("absent_node", "absent_node")), + assertEquals(Set.of(new VotingConfigExclusion("", "absent_node")), clusterService.getClusterApplierService().state().getVotingConfigExclusions()); } @@ -278,7 +279,7 @@ public void testTimesOut() throws InterruptedException { final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}, TimeValue.timeValueMillis(100)), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}, null, null, TimeValue.timeValueMillis(100)), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java index d213701d7e5e0..5124f7d8c2bfb 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.rest.action.admin.cluster; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.RestActionTestCase; @@ -49,20 +50,42 @@ public void testResolveVotingConfigExclusionsRequest() { AddVotingConfigExclusionsRequest addVotingConfigExclusionsRequest = action.resolveVotingConfigExclusionsRequest(deprecatedRequest); String[] expected = {"node-1","node-2", "node-3"}; assertArrayEquals(expected, addVotingConfigExclusionsRequest.getNodeDescriptions()); + assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeIds()); + assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeNames()); + assertWarnings("Using [node_name] for adding voting config exclustion will be removed in a future version. " + + "Please use [node_ids] or [node_names] instead"); } - public void testResolveVotingConfigExclusionsRequestNodeIdsOrNames() { + public void testResolveVotingConfigExclusionsRequestNodeIds() { Map params = new HashMap<>(); - params.put("node_ids_or_names", "node-1,node-2,node-3"); - RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()) + params.put("node_ids", "node-1,node-2,node-3"); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) .withMethod(RestRequest.Method.PUT) - .withPath("/_cluster/voting_config_exclusions/node_ids_or_names") + .withPath("/_cluster/voting_config_exclusions") .withParams(params) .build(); - AddVotingConfigExclusionsRequest addVotingConfigExclusionsRequest = action.resolveVotingConfigExclusionsRequest(deprecatedRequest); + AddVotingConfigExclusionsRequest addVotingConfigExclusionsRequest = action.resolveVotingConfigExclusionsRequest(request); String[] expected = {"node-1","node-2", "node-3"}; - assertArrayEquals(expected, addVotingConfigExclusionsRequest.getNodeDescriptions()); + assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeDescriptions()); + assertArrayEquals(expected, addVotingConfigExclusionsRequest.getNodeIds()); + assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeNames()); + } + + public void testResolveVotingConfigExclusionsRequestNodeNames() { + Map params = new HashMap<>(); + params.put("node_names", "node-1,node-2,node-3"); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(RestRequest.Method.PUT) + .withPath("/_cluster/voting_config_exclusions") + .withParams(params) + .build(); + + AddVotingConfigExclusionsRequest addVotingConfigExclusionsRequest = action.resolveVotingConfigExclusionsRequest(request); + String[] expected = {"node-1","node-2", "node-3"}; + assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeDescriptions()); + assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeIds()); + assertArrayEquals(expected, addVotingConfigExclusionsRequest.getNodeNames()); } } From ad415731524169e2f745ff3fff302aa67f04abfa Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Tue, 3 Mar 2020 23:23:06 -0800 Subject: [PATCH 04/29] Address feedback --- .../AddVotingConfigExclusionsRequest.java | 100 +++++++++++++----- .../coordination/CoordinationMetaData.java | 3 +- .../cluster/node/DiscoveryNodes.java | 58 ---------- .../RestAddVotingConfigExclusionAction.java | 32 +----- ...AddVotingConfigExclusionsRequestTests.java | 14 ++- ...tAddVotingConfigExclusionsActionTests.java | 96 ++++++++++++++++- 6 files changed, 188 insertions(+), 115 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 759c920bbd7e8..729cecbdbedbf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -34,6 +34,9 @@ import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -43,9 +46,11 @@ */ public class AddVotingConfigExclusionsRequest extends MasterNodeRequest { private final String[] nodeDescriptions; - private String[] nodeIds = null; - private String[] nodeNames = null; + private final String[] nodeIds; + private final String[] nodeNames; private final TimeValue timeout; + public static final String nodeDescriptionDeprecationMsg = "Using [node_name] for adding voting config exclustion will be removed " + + "in a future version. Please use [node_ids] or [node_names] instead"; /** * Construct a request to add voting config exclusions for master-eligible nodes matching the given descriptions, and wait for a @@ -66,6 +71,12 @@ public AddVotingConfigExclusionsRequest(String[] nodeDescriptions, String[] node if (timeout.compareTo(TimeValue.ZERO) < 0) { throw new IllegalArgumentException("timeout [" + timeout + "] must be non-negative"); } + + if(noneOrMoreThanOneIsSet(nodeDescriptions, nodeIds, nodeNames)) { + throw new IllegalArgumentException("Please set node identifiers correctly. " + + "One and only one of [node_name], [node_names] and [node_ids] has to be set"); + } + this.nodeDescriptions = nodeDescriptions; this.nodeIds = nodeIds; this.nodeNames = nodeNames; @@ -76,11 +87,14 @@ public AddVotingConfigExclusionsRequest(StreamInput in) throws IOException { super(in); // TODO should this be removed in the latest version where nodeIds and nodeNames are used? nodeDescriptions = in.readStringArray(); - // TODO which version to use here? - if (in.getVersion() == Version.V_EMPTY) { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { nodeIds = in.readStringArray(); nodeNames = in.readStringArray(); } + else { + nodeIds = Strings.EMPTY_ARRAY; + nodeNames = Strings.EMPTY_ARRAY; + } timeout = in.readTimeValue(); } @@ -98,32 +112,50 @@ Set resolveVotingConfigExclusions(ClusterState currentSta } } else { - DiscoveryNodes.NodeResolutionResults nodeResolutionResults; - Set resolvedNodes; - Set unresolvedNodes; + Set resolvedVotingConfigExclusions; + Set unresolvedVotingConfigExclusions; - if (nodeIds.length >= 1) { - nodeResolutionResults = allNodes.resolveNodesExact(true, nodeIds); + Set resolvedNodes = new HashSet<>(nodeIds.length); + Set unresolvedNodes = new HashSet<>(); - unresolvedNodes = Arrays.stream(nodeResolutionResults.getUnresolvedNodes()) - .map(nodeId -> new VotingConfigExclusion(nodeId, "")).collect(Collectors.toSet()); + if (nodeIds.length >= 1) { + for (String nodeId : nodeIds) { + if (allNodes.nodeExists(nodeId)) { + resolvedNodes.add(nodeId); + } + else { + unresolvedNodes.add(nodeId); + } + } + + unresolvedVotingConfigExclusions = unresolvedNodes.stream() + .map(nodeId -> new VotingConfigExclusion(nodeId, "_absent_")).collect(Collectors.toSet()); } else { - nodeResolutionResults = allNodes.resolveNodesExact(false, nodeNames); - - unresolvedNodes = Arrays.stream(nodeResolutionResults.getUnresolvedNodes()) - .map(nodeName -> new VotingConfigExclusion("", nodeName)).collect(Collectors.toSet()); + Map existingNodesNameId = new HashMap<>(); + for (DiscoveryNode node : allNodes) { + if (node.isMasterNode()) { + existingNodesNameId.put(node.getName(), node.getId()); + } + } + + for (String nodeName : nodeNames) { + if (existingNodesNameId.containsKey(nodeName)){ + resolvedNodes.add(existingNodesNameId.get(nodeName)); + } + else { + unresolvedNodes.add(nodeName); + } + } + + unresolvedVotingConfigExclusions = unresolvedNodes.stream() + .map(nodeName -> new VotingConfigExclusion("_absent_", nodeName)).collect(Collectors.toSet()); } - resolvedNodes = Arrays.stream(nodeResolutionResults.getResolvedNodes()) + resolvedVotingConfigExclusions = resolvedNodes.stream() .map(allNodes::get).filter(DiscoveryNode::isMasterNode).map(VotingConfigExclusion::new).collect(Collectors.toSet()); - allProcessedNodes = Sets.newHashSet(Iterables.concat(resolvedNodes, unresolvedNodes)); - - if (allProcessedNodes.isEmpty()) { - throw new IllegalArgumentException("add voting config exclusions request for nodeIds " + Arrays.asList(nodeIds) + - " or nodeNames " + Arrays.asList(nodeNames) + " matched no master-eligible nodes or absent nodes"); - } + allProcessedNodes = Sets.newHashSet(Iterables.concat(resolvedVotingConfigExclusions, unresolvedVotingConfigExclusions)); } allProcessedNodes.removeIf(n -> currentState.getVotingConfigExclusions().contains(n)); @@ -145,6 +177,26 @@ Set resolveVotingConfigExclusionsAndCheckMaximum(ClusterS return resolvedExclusions; } + public static boolean noneOrMoreThanOneIsSet(String[] deprecatedNodeDescription, String[] nodeIds, String[] nodeNames) { + if(arrayHasElement(deprecatedNodeDescription)) { + return arrayHasElement(nodeIds) || arrayHasElement(nodeNames); + } + else if (arrayHasElement(nodeIds)) { + return arrayHasElement(nodeNames); + } + else if (arrayHasElement(nodeNames)) { + return false; + } + else { + // none of the node identifiers are set + return true; + } + } + + private static boolean arrayHasElement(String[] array) { + return array != null && array.length > 0; + } + /** * @return descriptions of the nodes for whom to add voting config exclusions. */ @@ -183,11 +235,9 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); // TODO should this be removed in the latest version where nodeIds and nodeNames are used? out.writeStringArray(nodeDescriptions); - // TODO which version to use here? - if (out.getVersion() == Version.V_EMPTY) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeStringArray(nodeIds); out.writeStringArray(nodeNames); - } out.writeTimeValue(timeout); } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java index 340bfba003f6d..4d6483df61449 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java @@ -233,8 +233,7 @@ public static class VotingConfigExclusion implements Writeable, ToXContentFragme private final String nodeName; public VotingConfigExclusion(DiscoveryNode node) { - this.nodeId = node.getId(); - this.nodeName = node.getName(); + this(node.getId(), node.getName()); } public VotingConfigExclusion(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 8f59b1b965c84..22600b9f48ccb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -318,63 +317,6 @@ public DiscoveryNode resolveNode(String node) { return nodes.get(resolvedNodeIds[0]); } - public static class NodeResolutionResults { - private String[] resolvedNodes; - private String[] unresolvedNodes; - - public NodeResolutionResults(String[] resolvedNodes, String[] unresolvedNodes) { - this.resolvedNodes = resolvedNodes; - this.unresolvedNodes = unresolvedNodes; - } - - public String[] getResolvedNodes() { - return resolvedNodes; - } - - public String[] getUnresolvedNodes() { - return unresolvedNodes; - } - } - - public NodeResolutionResults resolveNodesExact(boolean isNodeIds, String... nodes) { - if (nodes == null || nodes.length == 0) { - return new NodeResolutionResults(StreamSupport.stream(this.spliterator(), false) - .map(DiscoveryNode::getId).toArray(String[]::new), new String[0]); - } else { - ObjectHashSet resolvedNodes = new ObjectHashSet<>(nodes.length); - ObjectHashSet unresolvedNodes = new ObjectHashSet<>(); - - if (isNodeIds) { - for (String nodeId : nodes) { - if (nodeExists(nodeId)) { - resolvedNodes.add(nodeId); - } - else { - unresolvedNodes.add(nodeId); - } - } - } - else { - Map existingNodesNameId = new HashMap<>(); - for (DiscoveryNode node : this) { - existingNodesNameId.put(node.getName(), node.getId()); - } - - for (String nodeName : nodes) { - if (existingNodesNameId.containsKey(nodeName)){ - resolvedNodes.add(existingNodesNameId.get(nodeName)); - } - else { - unresolvedNodes.add(nodeName); - } - } - } - - return new NodeResolutionResults(resolvedNodes.toArray(String.class), unresolvedNodes.toArray(String.class)); - } - } - - /** * resolves a set of node "descriptions" to concrete and existing node ids. "descriptions" can be (resolved in this order): * - "_local" or "_master" for the relevant nodes diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java index 075dab3a64c5f..7d96ac7321bfd 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java @@ -34,6 +34,8 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import static org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest.nodeDescriptionDeprecationMsg; + public class RestAddVotingConfigExclusionAction extends BaseRestHandler { private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(30L); @@ -41,8 +43,8 @@ public class RestAddVotingConfigExclusionAction extends BaseRestHandler { private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); public RestAddVotingConfigExclusionAction(RestController controller) { - // TODO This API is being deprecated. - controller.registerHandler(RestRequest.Method.POST, "/_cluster/voting_config_exclusions/{node_name}", this); + controller.registerAsDeprecatedHandler(RestRequest.Method.POST, "/_cluster/voting_config_exclusions/{node_name}", this, + nodeDescriptionDeprecationMsg, DEPRECATION_LOGGER); controller.registerHandler(RestRequest.Method.POST, "/_cluster/voting_config_exclusions", this); } @@ -68,9 +70,6 @@ AddVotingConfigExclusionsRequest resolveVotingConfigExclusionsRequest(final Rest String nodeNames = null; if (request.hasParam("node_name")) { - DEPRECATION_LOGGER.deprecatedAndMaybeLog("add_voting_config_exclusion", - "Using [node_name] for adding voting config exclustion will be removed in a future version. " + - "Please use [node_ids] or [node_names] instead"); deprecatedNodeDescription = request.param("node_name"); } @@ -79,12 +78,7 @@ AddVotingConfigExclusionsRequest resolveVotingConfigExclusionsRequest(final Rest } if (request.hasParam("node_names")){ - nodeNames = request.param("node_names"); - } - - if(!oneAndonlyOneIsSet(deprecatedNodeDescription, nodeIds, nodeNames)) { - throw new IllegalArgumentException("Please set node identifiers correctly. " + - "One and only one of [node_name], [node_names] and [node_ids] has to be set"); + nodeNames = request.param("node_names"); } return new AddVotingConfigExclusionsRequest( @@ -95,21 +89,5 @@ AddVotingConfigExclusionsRequest resolveVotingConfigExclusionsRequest(final Rest ); } - private boolean oneAndonlyOneIsSet(String deprecatedNodeDescription, String nodeIds, String nodeNames) { - if(Strings.hasText(deprecatedNodeDescription)) { - return Strings.isNullOrEmpty(nodeIds) && Strings.isNullOrEmpty(nodeNames); - } - else if (Strings.hasText(nodeIds)) { - return Strings.isNullOrEmpty(nodeNames); - } - else if (Strings.hasText(nodeNames)) { - return true; - } - else { - // none of the node identifiers are set - return false; - } - - } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index dfca53764e226..02ea9da3b9258 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -37,6 +37,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; public class AddVotingConfigExclusionsRequestTests extends ESTestCase { @@ -88,6 +89,16 @@ public void testResolve() { assertThat(makeRequest().resolveVotingConfigExclusions(clusterState), containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); + assertThat(makeRequest("_all").resolveVotingConfigExclusions(clusterState), + containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); + assertThat(makeRequest("_local").resolveVotingConfigExclusions(clusterState), + contains(localNodeExclusion)); + assertThat(makeRequest("other*").resolveVotingConfigExclusions(clusterState), + containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + + assertThat(expectThrows(IllegalArgumentException.class, + () -> makeRequest("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), + equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); } public void testResolveAndCheckMaximum() { @@ -124,7 +135,8 @@ public void testResolveAndCheckMaximum() { assertThat(makeRequest().resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 3, "setting.name"), containsInAnyOrder(localNodeExclusion, otherNode2Exclusion)); - + assertThat(makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), + contains(localNodeExclusion)); assertThat(expectThrows(IllegalArgumentException.class, () -> makeRequest().resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name")).getMessage(), equalTo("add voting config exclusions request for [] would add [2] exclusions to the existing [1] which would exceed " + diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 107fd6e75d3a3..343739ad24122 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -167,6 +167,57 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); } + + public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + + clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(new String[]{"other*"}), + expectSuccess(r -> { + assertNotNull(r); + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), + containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + } + + public void testWithdrawsVotesFromAllMasterEligibleNodes() throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + + clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(new String[]{"_all"}), + expectSuccess(r -> { + assertNotNull(r); + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), + containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); + } + + public void testWithdrawsVoteFromLocalNode() throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + + clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(new String[]{"_local"}), + expectSuccess(r -> { + assertNotNull(r); + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), + contains(localNodeExclusion)); + } public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedException { final ClusterState state = clusterService.state(); setState(clusterService, builder(state) @@ -180,7 +231,7 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc // no observer to reconfigure transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}, null, null, TimeValue.ZERO), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.ZERO), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -192,6 +243,46 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc contains(otherNode1Exclusion)); } + public void testReturnsErrorIfNoMatchingNodes() throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + final SetOnce exceptionHolder = new SetOnce<>(); + + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(new String[]{"not-a-node"}), + expectError(e -> { + exceptionHolder.set(e); + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + final Throwable rootCause = exceptionHolder.get().getRootCause(); + assertThat(rootCause, instanceOf(IllegalArgumentException.class)); + assertThat(rootCause.getMessage(), + equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); + } + + public void testOnlyMatchesMasterEligibleNodes() throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + final SetOnce exceptionHolder = new SetOnce<>(); + + + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(new String[]{"_all", "master:false"}), + expectError(e -> { + exceptionHolder.set(e); + countDownLatch.countDown(); + }) + ); + + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + final Throwable rootCause = exceptionHolder.get().getRootCause(); + assertThat(rootCause, instanceOf(IllegalArgumentException.class)); + assertThat(rootCause.getMessage(), + equalTo("add voting config exclusions request for [_all, master:false] matched no master-eligible nodes")); + } + public void testMatchesAbsentNodesByNodeNames() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); @@ -279,7 +370,8 @@ public void testTimesOut() throws InterruptedException { final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}, null, null, TimeValue.timeValueMillis(100)), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + TimeValue.timeValueMillis(100)), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); From b085c2cd8980b504ba19c1ce89c4513f9ab143f5 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Tue, 3 Mar 2020 23:57:50 -0800 Subject: [PATCH 05/29] Address comment --- .../AddVotingConfigExclusionsRequestTests.java | 8 -------- .../TransportAddVotingConfigExclusionsActionTests.java | 2 +- .../cluster/RestAddVotingConfigExclusionActionTests.java | 2 -- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index 02ea9da3b9258..c1e1534d3b762 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -87,8 +87,6 @@ public void testResolve() { final ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")).nodes(new Builder() .add(localNode).add(otherNode1).add(otherNode2).add(otherDataNode).localNodeId(localNode.getId())).build(); - assertThat(makeRequest().resolveVotingConfigExclusions(clusterState), - containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); assertThat(makeRequest("_all").resolveVotingConfigExclusions(clusterState), containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); assertThat(makeRequest("_local").resolveVotingConfigExclusions(clusterState), @@ -133,14 +131,8 @@ public void testResolveAndCheckMaximum() { .coordinationMetaData(CoordinationMetaData.builder().addVotingConfigExclusion(otherNode1Exclusion).build())); final ClusterState clusterState = builder.build(); - assertThat(makeRequest().resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 3, "setting.name"), - containsInAnyOrder(localNodeExclusion, otherNode2Exclusion)); assertThat(makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), contains(localNodeExclusion)); - assertThat(expectThrows(IllegalArgumentException.class, - () -> makeRequest().resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name")).getMessage(), - equalTo("add voting config exclusions request for [] would add [2] exclusions to the existing [1] which would exceed " + - "the maximum of [2] set by [setting.name]")); assertThat(expectThrows(IllegalArgumentException.class, () -> makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")).getMessage(), equalTo("add voting config exclusions request for [_local] would add [1] exclusions to the existing [1] which would " + diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 343739ad24122..18849211ac16c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -294,7 +294,7 @@ public void testMatchesAbsentNodesByNodeNames() throws InterruptedException { ); assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - assertEquals(Set.of(new VotingConfigExclusion("", "absent_node")), + assertEquals(Set.of(new VotingConfigExclusion("_absent_", "absent_node")), clusterService.getClusterApplierService().state().getVotingConfigExclusions()); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java index 5124f7d8c2bfb..d184e86a044c5 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java @@ -52,8 +52,6 @@ public void testResolveVotingConfigExclusionsRequest() { assertArrayEquals(expected, addVotingConfigExclusionsRequest.getNodeDescriptions()); assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeIds()); assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeNames()); - assertWarnings("Using [node_name] for adding voting config exclustion will be removed in a future version. " + - "Please use [node_ids] or [node_names] instead"); } public void testResolveVotingConfigExclusionsRequestNodeIds() { From 57cbc474394df4f442e849dc953ad1281b6c430a Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 4 Mar 2020 19:14:52 -0800 Subject: [PATCH 06/29] Update nodeId for VotingConfigExclusion when node with matching name joins --- .../AddVotingConfigExclusionsRequest.java | 6 ++-- .../coordination/CoordinationMetaData.java | 1 + .../coordination/JoinTaskExecutor.java | 31 ++++++++++++++++++- ...tAddVotingConfigExclusionsActionTests.java | 2 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 729cecbdbedbf..0f561de9d449d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -129,7 +129,8 @@ Set resolveVotingConfigExclusions(ClusterState currentSta } unresolvedVotingConfigExclusions = unresolvedNodes.stream() - .map(nodeId -> new VotingConfigExclusion(nodeId, "_absent_")).collect(Collectors.toSet()); + .map(nodeId -> new VotingConfigExclusion(nodeId, VotingConfigExclusion.MISSING_VALUE_MARKER)) + .collect(Collectors.toSet()); } else { Map existingNodesNameId = new HashMap<>(); @@ -149,7 +150,8 @@ Set resolveVotingConfigExclusions(ClusterState currentSta } unresolvedVotingConfigExclusions = unresolvedNodes.stream() - .map(nodeName -> new VotingConfigExclusion("_absent_", nodeName)).collect(Collectors.toSet()); + .map(nodeName -> new VotingConfigExclusion(VotingConfigExclusion.MISSING_VALUE_MARKER, nodeName)) + .collect(Collectors.toSet()); } resolvedVotingConfigExclusions = resolvedNodes.stream() diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java index 4d6483df61449..02eccf8f1f9fa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java @@ -229,6 +229,7 @@ public CoordinationMetaData build() { } public static class VotingConfigExclusion implements Writeable, ToXContentFragment { + public static final String MISSING_VALUE_MARKER = "_absent_"; private final String nodeId; private final String nodeName; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java index e3cf0cb2558b2..82bfd51ee4189 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java @@ -37,8 +37,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.BiConsumer; +import java.util.stream.Collectors; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -124,6 +128,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo // we only enforce major version transitions on a fully formed clusters final boolean enforceMajorVersion = currentState.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false; // processing any joins + Map joiniedNodeNameIds = new HashMap<>(); for (final Task joinTask : joiningNodes) { if (joinTask.isBecomeMasterTask() || joinTask.isFinishElectionTask()) { // noop @@ -143,6 +148,9 @@ public ClusterTasksResult execute(ClusterState currentState, List jo nodesChanged = true; minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion()); maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion()); + // TODO do we need to check node.isMasterNode here for eligibility? I think since in the checks later master + // eligibility will be taken into account anyway, here we may not need the check? + joiniedNodeNameIds.put(node.getName(), node.getId()); } catch (IllegalArgumentException | IllegalStateException e) { results.failure(joinTask, e); continue; @@ -150,12 +158,33 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } results.success(joinTask); } + if (nodesChanged) { rerouteService.reroute("post-join reroute", Priority.HIGH, ActionListener.wrap( r -> logger.trace("post-join reroute completed"), e -> logger.debug("post-join reroute failed", e))); - return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).build())); + if (joiniedNodeNameIds.isEmpty() == false) { + Set currentVotingConfigExclusions = currentState.getVotingConfigExclusions(); + Set newVotingConfigExclusions = currentVotingConfigExclusions.stream() + .map(e -> { + // Update nodeId in VotingConfigExclusion when a new node with excluded node name joins + if (CoordinationMetaData.VotingConfigExclusion.MISSING_VALUE_MARKER.equals(e.getNodeId()) && + joiniedNodeNameIds.containsKey(e.getNodeName())) { + return new CoordinationMetaData.VotingConfigExclusion(joiniedNodeNameIds.get(e.getNodeName()), e.getNodeName()); + } else { + return e; + } + }).collect(Collectors.toSet()); + + CoordinationMetaData.Builder coordMetaDataBuilder = CoordinationMetaData.builder(currentState.coordinationMetaData()) + .clearVotingConfigExclusions(); + newVotingConfigExclusions.forEach(coordMetaDataBuilder::addVotingConfigExclusion); + MetaData newMetaData = MetaData.builder(currentState.metaData()).coordinationMetaData(coordMetaDataBuilder.build()).build(); + return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).metaData(newMetaData).build())); + } else { + return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).build())); + } } else { // we must return a new cluster state instance to force publishing. This is important // for the joining node to finalize its join and set us as a master diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 18849211ac16c..171eaee74fe27 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -294,7 +294,7 @@ public void testMatchesAbsentNodesByNodeNames() throws InterruptedException { ); assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); - assertEquals(Set.of(new VotingConfigExclusion("_absent_", "absent_node")), + assertEquals(Set.of(new VotingConfigExclusion(VotingConfigExclusion.MISSING_VALUE_MARKER, "absent_node")), clusterService.getClusterApplierService().state().getVotingConfigExclusions()); } From e08a8a15a34a8ed138b62ac1115f9232323a41dd Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 5 Mar 2020 20:07:38 -0800 Subject: [PATCH 07/29] Add test cases to AddVotingConfigExclusionsRequestTests --- .../AddVotingConfigExclusionsRequest.java | 4 +- ...AddVotingConfigExclusionsRequestTests.java | 193 +++++++++++++++++- 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 0f561de9d449d..21bb9f0d872be 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -179,7 +179,7 @@ Set resolveVotingConfigExclusionsAndCheckMaximum(ClusterS return resolvedExclusions; } - public static boolean noneOrMoreThanOneIsSet(String[] deprecatedNodeDescription, String[] nodeIds, String[] nodeNames) { + private boolean noneOrMoreThanOneIsSet(String[] deprecatedNodeDescription, String[] nodeIds, String[] nodeNames) { if(arrayHasElement(deprecatedNodeDescription)) { return arrayHasElement(nodeIds) || arrayHasElement(nodeNames); } @@ -195,7 +195,7 @@ else if (arrayHasElement(nodeNames)) { } } - private static boolean arrayHasElement(String[] array) { + private boolean arrayHasElement(String[] array) { return array != null && array.length > 0; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index c1e1534d3b762..119757064aad2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -42,7 +42,7 @@ public class AddVotingConfigExclusionsRequestTests extends ESTestCase { public void testSerialization() throws IOException { - int descriptionCount = between(0, 5); + int descriptionCount = between(1, 5); String[] descriptions = new String[descriptionCount]; for (int i = 0; i < descriptionCount; i++) { descriptions[i] = randomAlphaOfLength(10); @@ -56,6 +56,30 @@ public void testSerialization() throws IOException { assertThat(deserialized.getTimeout(), equalTo(originalRequest.getTimeout())); } + public void testSerializationForNodeIdOrNodeName() throws IOException { + // TODO still need adjustment for version? copyWriteable uses Version.CURRENT + AddVotingConfigExclusionsRequest originalRequest = new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, + new String[]{"nodeId1", "nodeId2"}, Strings.EMPTY_ARRAY, TimeValue.ZERO); + AddVotingConfigExclusionsRequest deserialized = copyWriteable(originalRequest, writableRegistry(), + AddVotingConfigExclusionsRequest::new); + + assertThat(deserialized.getNodeDescriptions(), equalTo(originalRequest.getNodeDescriptions())); + assertThat(deserialized.getNodeIds(), equalTo(originalRequest.getNodeIds())); + assertThat(deserialized.getNodeNames(), equalTo(originalRequest.getNodeNames())); + assertThat(deserialized.getTimeout(), equalTo(originalRequest.getTimeout())); + + // TODO still need adjustment for version? copyWriteable uses Version.CURRENT + originalRequest = new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, new String[]{"nodeName1", "nodeName2"}, TimeValue.ZERO); + deserialized = copyWriteable(originalRequest, writableRegistry(), + AddVotingConfigExclusionsRequest::new); + + assertThat(deserialized.getNodeDescriptions(), equalTo(originalRequest.getNodeDescriptions())); + assertThat(deserialized.getNodeIds(), equalTo(originalRequest.getNodeIds())); + assertThat(deserialized.getNodeNames(), equalTo(originalRequest.getNodeNames())); + assertThat(deserialized.getTimeout(), equalTo(originalRequest.getTimeout())); + } + public void testResolve() { final DiscoveryNode localNode = new DiscoveryNode( "local", @@ -99,6 +123,173 @@ public void testResolve() { equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); } + public void testResolveAllNodeIdentifiersNullOrEmpty() { + assertThat(expectThrows(IllegalArgumentException.class, + () -> new AddVotingConfigExclusionsRequest(null, null, null, TimeValue.ZERO)).getMessage(), + equalTo("Please set node identifiers correctly. " + + "One and only one of [node_name], [node_names] and [node_ids] has to be set")); + + assertThat(expectThrows(IllegalArgumentException.class, + () -> new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.ZERO)) + .getMessage(), equalTo("Please set node identifiers correctly. " + + "One and only one of [node_name], [node_names] and [node_ids] has to be set")); + } + + public void testResolveMoreThanOneNodeIdentifiersSet() { + assertThat(expectThrows(IllegalArgumentException.class, + () -> new AddVotingConfigExclusionsRequest(new String[]{"local"}, new String[]{"nodeId"}, Strings.EMPTY_ARRAY, TimeValue.ZERO)) + .getMessage(), equalTo("Please set node identifiers correctly. " + + "One and only one of [node_name], [node_names] and [node_ids] has to be set")); + + assertThat(expectThrows(IllegalArgumentException.class, + () -> new AddVotingConfigExclusionsRequest(new String[]{"local"}, Strings.EMPTY_ARRAY, + new String[]{"nodeName"}, TimeValue.ZERO)) + .getMessage(), equalTo("Please set node identifiers correctly. " + + "One and only one of [node_name], [node_names] and [node_ids] has to be set")); + + assertThat(expectThrows(IllegalArgumentException.class, + () -> new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[]{"nodeId"}, + new String[]{"nodeName"}, TimeValue.ZERO)) + .getMessage(), equalTo("Please set node identifiers correctly. " + + "One and only one of [node_name], [node_names] and [node_ids] has to be set")); + + assertThat(expectThrows(IllegalArgumentException.class, + () -> new AddVotingConfigExclusionsRequest(new String[]{"local"}, new String[]{"nodeId"}, + new String[]{"nodeName"}, TimeValue.ZERO)) + .getMessage(), equalTo("Please set node identifiers correctly. " + + "One and only one of [node_name], [node_names] and [node_ids] has to be set")); + } + + public void testResolveByNodeIds() { + final DiscoveryNode node1 = new DiscoveryNode( + "nodeName1", + "nodeId1", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + final VotingConfigExclusion node1Exclusion = new VotingConfigExclusion(node1); + + final DiscoveryNode node2 = new DiscoveryNode( + "nodeName2", + "nodeId2", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + final VotingConfigExclusion node2Exclusion = new VotingConfigExclusion(node2); + + final DiscoveryNode node3 = new DiscoveryNode( + "nodeName3", + "nodeId3", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + + final VotingConfigExclusion unresolvableVotingConfigExclusion = new VotingConfigExclusion("unresolvableNodeId", + VotingConfigExclusion.MISSING_VALUE_MARKER); + + final ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")) + .nodes(new Builder().add(node1).add(node2).add(node3).localNodeId(node1.getId())).build(); + + assertThat(new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[]{"nodeId1", "nodeId2"}, + Strings.EMPTY_ARRAY, TimeValue.ZERO).resolveVotingConfigExclusions(clusterState), + containsInAnyOrder(node1Exclusion, node2Exclusion)); + + assertThat(new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[]{"nodeId1", "unresolvableNodeId"}, + Strings.EMPTY_ARRAY, TimeValue.ZERO).resolveVotingConfigExclusions(clusterState), + containsInAnyOrder(node1Exclusion, unresolvableVotingConfigExclusion)); + } + + + public void testResolveByNodeNames() { + final DiscoveryNode node1 = new DiscoveryNode( + "nodeName1", + "nodeId1", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + final VotingConfigExclusion node1Exclusion = new VotingConfigExclusion(node1); + + final DiscoveryNode node2 = new DiscoveryNode( + "nodeName2", + "nodeId2", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + final VotingConfigExclusion node2Exclusion = new VotingConfigExclusion(node2); + + final DiscoveryNode node3 = new DiscoveryNode( + "nodeName3", + "nodeId3", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + + final VotingConfigExclusion unresolvableVotingConfigExclusion = new VotingConfigExclusion( + VotingConfigExclusion.MISSING_VALUE_MARKER, "unresolvableNodeName"); + + final ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")) + .nodes(new Builder().add(node1).add(node2).add(node3).localNodeId(node1.getId())).build(); + + assertThat(new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{"nodeName1", "nodeName2"}, TimeValue.ZERO) + .resolveVotingConfigExclusions(clusterState), + containsInAnyOrder(node1Exclusion, node2Exclusion)); + + assertThat(new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{"nodeName1", "unresolvableNodeName"}, TimeValue.ZERO) + .resolveVotingConfigExclusions(clusterState), + containsInAnyOrder(node1Exclusion, unresolvableVotingConfigExclusion)); + } + + public void testResolveRemoveExistingVotingConfigExclusions() { + final DiscoveryNode node1 = new DiscoveryNode( + "nodeName1", + "nodeId1", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + final VotingConfigExclusion node1Exclusion = new VotingConfigExclusion(node1); + + final DiscoveryNode node2 = new DiscoveryNode( + "nodeName2", + "nodeId2", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + final VotingConfigExclusion node2Exclusion = new VotingConfigExclusion(node2); + + final DiscoveryNode node3 = new DiscoveryNode( + "nodeName3", + "nodeId3", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); + + final VotingConfigExclusion existingVotingConfigExclusion = new VotingConfigExclusion("nodeId1", "nodeName1"); + + MetaData metaData = MetaData.builder() + .coordinationMetaData(CoordinationMetaData.builder() + .addVotingConfigExclusion(existingVotingConfigExclusion).build()) + .build(); + + final ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")).metaData(metaData) + .nodes(new Builder().add(node1).add(node2).add(node3).localNodeId(node1.getId())).build(); + + assertThat(new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[]{"nodeId1", "nodeId2"}, + Strings.EMPTY_ARRAY, TimeValue.ZERO) + .resolveVotingConfigExclusions(clusterState), + contains(node2Exclusion)); + } + public void testResolveAndCheckMaximum() { final DiscoveryNode localNode = new DiscoveryNode( "local", From d32512355d379374ae40cd8b7f1d7e0fd2a207cb Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Fri, 6 Mar 2020 19:21:01 -0800 Subject: [PATCH 08/29] Add assertion for voting config exclusion in cluster state --- .../cluster/coordination/Coordinator.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 20b3637680fb9..1d11a61248f81 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -868,6 +868,8 @@ assert localNodeMayWinElection(getLastAcceptedState()) : // Package-private for testing ClusterState improveConfiguration(ClusterState clusterState) { assert Thread.holdsLock(mutex) : "Coordinator mutex not held"; + assert validVotingConfigExclusionState(clusterState) : "Voting Config Exclusion in invalid state. " + + "Exclusions may not be processed correctly"; // exclude any nodes whose ID is in the voting config exclusions list ... final Stream excludedNodeIds = clusterState.getVotingConfigExclusions().stream().map(VotingConfigExclusion::getNodeId); @@ -895,6 +897,31 @@ ClusterState improveConfiguration(ClusterState clusterState) { return clusterState; } + /* + * Valid Voting Configuration Exclusion state criteria: + * 1. Every voting config exclusion with an ID of _absent_ should not match any nodes currently in the cluster by name + * 2. Every voting config exclusion with a name of _absent_ should not match any nodes currently in the cluster by ID + */ + private boolean validVotingConfigExclusionState(ClusterState clusterState) { + Set votingConfigExclusions = clusterState.getVotingConfigExclusions(); + Set nodeNamesWithAbsentId = votingConfigExclusions.stream() + .filter(e -> e.getNodeId().equals(VotingConfigExclusion.MISSING_VALUE_MARKER)) + .map(VotingConfigExclusion::getNodeName) + .collect(Collectors.toSet()); + Set nodeIdsWithAbsentName = votingConfigExclusions.stream() + .filter(e -> e.getNodeName().equals(VotingConfigExclusion.MISSING_VALUE_MARKER)) + .map(VotingConfigExclusion::getNodeId) + .collect(Collectors.toSet()); + for (DiscoveryNode node : clusterState.getNodes()) { + // TODO should this check be applied to master-eligible nodes only? + if(nodeIdsWithAbsentName.contains(node.getId()) || nodeNamesWithAbsentId.contains(node.getName())) { + return false; + } + } + + return true; + } + private AtomicBoolean reconfigurationTaskScheduled = new AtomicBoolean(); private void scheduleReconfigurationIfNeeded() { From bede9a011b95a8b838bbd51d6053726e1f008932 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Fri, 6 Mar 2020 20:12:20 -0800 Subject: [PATCH 09/29] Add test cases to TransportAddVotingConfigExclusionsActionTests --- .../AddVotingConfigExclusionsRequest.java | 3 +- ...tAddVotingConfigExclusionsActionTests.java | 113 +++++++++++++++++- 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 21bb9f0d872be..7426e982673d0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -110,8 +110,7 @@ Set resolveVotingConfigExclusions(ClusterState currentSta throw new IllegalArgumentException("add voting config exclusions request for " + Arrays.asList(nodeDescriptions) + " matched no master-eligible nodes"); } - } - else { + } else { Set resolvedVotingConfigExclusions; Set unresolvedVotingConfigExclusions; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 171eaee74fe27..09a16a1b5d1d2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -231,7 +231,7 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc // no observer to reconfigure transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.ZERO), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -283,11 +283,48 @@ public void testOnlyMatchesMasterEligibleNodes() throws InterruptedException { equalTo("add voting config exclusions request for [_all, master:false] matched no master-eligible nodes")); } - public void testMatchesAbsentNodesByNodeNames() throws InterruptedException { + public void testExcludeAbsentNodesByNodeIds() throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + + clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[]{"absent_id"}, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), + expectSuccess(e -> { + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertEquals(Set.of(new VotingConfigExclusion("absent_id", VotingConfigExclusion.MISSING_VALUE_MARKER)), + clusterService.getClusterApplierService().state().getVotingConfigExclusions()); + } + + public void testExcludeExistingNodesByNodeIds() throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + + clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[]{"other1", "other2"}, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), + expectSuccess(r -> { + assertNotNull(r); + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), + containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + } + + public void testExcludeAbsentNodesByNodeNames() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); + clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, new String[]{"absent_node"}, TimeValue.ZERO), + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{"absent_node"}, TimeValue.timeValueSeconds(30)), expectSuccess(e -> { countDownLatch.countDown(); }) @@ -298,6 +335,24 @@ public void testMatchesAbsentNodesByNodeNames() throws InterruptedException { clusterService.getClusterApplierService().state().getVotingConfigExclusions()); } + public void testExcludeExistingNodesByNodeNames() throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + + clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{"other1", "other2"}, TimeValue.timeValueSeconds(30)), + expectSuccess(r -> { + assertNotNull(r); + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), + containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + } + public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedException { final ClusterState state = clusterService.state(); final ClusterState.Builder builder = builder(state); @@ -323,6 +378,58 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce contains(otherNode1Exclusion)); } + public void testExcludeByNodeIdSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedException { + final ClusterState state = clusterService.state(); + final ClusterState.Builder builder = builder(state); + builder.metaData(MetaData.builder(state.metaData()). + coordinationMetaData( + CoordinationMetaData.builder(state.coordinationMetaData()) + .addVotingConfigExclusion(otherNode1Exclusion). + build())); + setState(clusterService, builder); + + final CountDownLatch countDownLatch = new CountDownLatch(1); + + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[]{"other1"}, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), + expectSuccess(r -> { + assertNotNull(r); + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), + contains(otherNode1Exclusion)); + } + + public void testExcludeByNodeNameSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedException { + final ClusterState state = clusterService.state(); + final ClusterState.Builder builder = builder(state); + builder.metaData(MetaData.builder(state.metaData()). + coordinationMetaData( + CoordinationMetaData.builder(state.coordinationMetaData()) + .addVotingConfigExclusion(otherNode1Exclusion). + build())); + setState(clusterService, builder); + + final CountDownLatch countDownLatch = new CountDownLatch(1); + + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{"other1"}, TimeValue.timeValueSeconds(30)), + expectSuccess(r -> { + assertNotNull(r); + countDownLatch.countDown(); + }) + ); + + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), + contains(otherNode1Exclusion)); + } + public void testReturnsErrorIfMaximumExclusionCountExceeded() throws InterruptedException { final MetaData.Builder metaDataBuilder = MetaData.builder(clusterService.state().metaData()).persistentSettings( Settings.builder().put(clusterService.state().metaData().persistentSettings()) From fb337f2efd5476fe0160356b45cda3a653de67ce Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Mon, 9 Mar 2020 19:53:49 -0700 Subject: [PATCH 10/29] Add test to NodeJoinTests --- .../cluster/coordination/NodeJoinTests.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java index 60575f7487676..ab1cc54f96fa9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java @@ -113,6 +113,8 @@ private static ClusterState initialState(DiscoveryNode localNode, long term, lon .term(term) .lastAcceptedConfiguration(config) .lastCommittedConfiguration(config) + .addVotingConfigExclusion(new CoordinationMetaData.VotingConfigExclusion( + CoordinationMetaData.VotingConfigExclusion.MISSING_VALUE_MARKER, "knownNodeName")) .build())) .version(version) .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build(); @@ -302,8 +304,7 @@ public void testJoinWithHigherTermButBetterStateGetsRejected() { assertFalse(isLocalNodeElectedMaster()); } - public void testJoinWithHigherTermButBetterStateStillElectsMasterThroughSelfJoin() { - DiscoveryNode node0 = newNode(0, true); + public void testJoinWithHigherTermButBetterStateStillElectsMasterThroughSelfJoin() { DiscoveryNode node0 = newNode(0, true); DiscoveryNode node1 = newNode(1, true); long initialTerm = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10); @@ -377,6 +378,27 @@ public void testJoinFollowerWithHigherTerm() throws Exception { assertTrue(isLocalNodeElectedMaster()); } + public void testJoinUpdateVotingConfigExclusion() throws Exception { + DiscoveryNode initialNode = newNode(0, true); + long initialTerm = randomLongBetween(1, 10); + long initialVersion = randomLongBetween(1, 10); + + setupFakeMasterServiceAndCoordinator(initialTerm, initialState(initialNode, initialTerm, initialVersion, + new VotingConfiguration(Collections.singleton(initialNode.getId())))); + + DiscoveryNode knownJoiningNode = new DiscoveryNode("knownNodeName", "newNodeId", buildNewFakeTransportAddress(), + emptyMap(), Set.of(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); + long newTerm = initialTerm + randomLongBetween(1, 10); + long newerTerm = newTerm + randomLongBetween(1, 10); + + joinNodeAndRun(new JoinRequest(knownJoiningNode, + Optional.of(new Join(knownJoiningNode, initialNode, newerTerm, initialTerm, initialVersion)))); + + assertTrue(MasterServiceTests.discoveryState(masterService).getVotingConfigExclusions().stream().anyMatch(exclusion -> { + return "knownNodeName".equals(exclusion.getNodeName()) && "newNodeId".equals(exclusion.getNodeId()); + })); + } + private void handleStartJoinFrom(DiscoveryNode node, long term) throws Exception { final RequestHandlerRegistry startJoinHandler = (RequestHandlerRegistry) transport.getRequestHandler(JoinHelper.START_JOIN_ACTION_NAME); From 5e28294d54d11a047ed376d1eb517591ab6fc11f Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Mon, 9 Mar 2020 21:20:34 -0700 Subject: [PATCH 11/29] Add test to CooridnatorTests --- .../coordination/CoordinatorTests.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 3004dc99ac360..e185a8c2ac94d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.coordination.Coordinator.Mode; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.regex.Regex; @@ -55,6 +56,8 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; import static org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.DEFAULT_DELAY_VARIABILITY; import static org.elasticsearch.cluster.coordination.Coordinator.Mode.CANDIDATE; import static org.elasticsearch.cluster.coordination.Coordinator.PUBLISH_TIMEOUT_SETTING; @@ -1385,4 +1388,53 @@ public void testDoesNotPerformElectionWhenRestartingFollower() { } } + public void testImprovecOnfigurationPerformsVotingConfigExclusionStateCheck() { + try (Cluster cluster = new Cluster(1)) { + cluster.runRandomly(); + cluster.stabilise(); + + final Coordinator coordinator = cluster.getAnyLeader().coordinator; + final ClusterState currentState = coordinator.getLastAcceptedState(); + + Set newVotingConfigExclusion1 = new HashSet<>(){{ + add(new CoordinationMetaData.VotingConfigExclusion("resolvableNodeId", + CoordinationMetaData.VotingConfigExclusion.MISSING_VALUE_MARKER)); + }}; + + ClusterState newState1 = buildNewClusterStateWithVotingConfigExclusion(currentState, newVotingConfigExclusion1); + + synchronized (coordinator.mutex) { + AssertionError error = expectThrows(AssertionError.class, () -> coordinator.improveConfiguration(newState1)); + assertEquals("Voting Config Exclusion in invalid state. Exclusions may not be processed correctly", + error.getMessage()); + } + + Set newVotingConfigExclusion2 = new HashSet<>(){{ + add(new CoordinationMetaData.VotingConfigExclusion(CoordinationMetaData.VotingConfigExclusion.MISSING_VALUE_MARKER, + "resolvableNodeName")); + }}; + + ClusterState newState2 = buildNewClusterStateWithVotingConfigExclusion(currentState, newVotingConfigExclusion1); + + synchronized (coordinator.mutex) { + AssertionError error = expectThrows(AssertionError.class, () -> coordinator.improveConfiguration(newState2)); + assertEquals("Voting Config Exclusion in invalid state. Exclusions may not be processed correctly", + error.getMessage()); + } + } + } + + private ClusterState buildNewClusterStateWithVotingConfigExclusion(ClusterState currentState, + Set newVotingConfigExclusion) { + DiscoveryNodes newNodes = DiscoveryNodes.builder(currentState.nodes()) + .add(new DiscoveryNode("resolvableNodeName", "resolvableNodeId", buildNewFakeTransportAddress(), + emptyMap(), emptySet(), Version.CURRENT)).build(); + + CoordinationMetaData.Builder coordMetaDataBuilder = CoordinationMetaData.builder(currentState.coordinationMetaData()); + newVotingConfigExclusion.forEach(coordMetaDataBuilder::addVotingConfigExclusion); + MetaData newMetaData = MetaData.builder(currentState.metaData()).coordinationMetaData(coordMetaDataBuilder.build()).build(); + + return ClusterState.builder(currentState).nodes(newNodes).metaData(newMetaData).build(); + } + } From 49f749c02cb7c2fda6f705f130e1e269f97e7703 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Mon, 9 Mar 2020 22:35:25 -0700 Subject: [PATCH 12/29] Inline deprecation message --- .../configuration/AddVotingConfigExclusionsRequest.java | 2 -- .../cluster/RestAddVotingConfigExclusionAction.java | 9 +++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 7426e982673d0..f78d4b87f8886 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -49,8 +49,6 @@ public class AddVotingConfigExclusionsRequest extends MasterNodeRequest Date: Wed, 25 Mar 2020 18:27:13 -0700 Subject: [PATCH 13/29] Address some comments that can be fixed quickly --- .../AddVotingConfigExclusionsRequest.java | 35 +++++++++++-------- .../cluster/coordination/Coordinator.java | 9 +++-- .../coordination/JoinTaskExecutor.java | 26 ++++++++------ .../cluster/node/DiscoveryNodes.java | 7 ++-- .../RestAddVotingConfigExclusionAction.java | 7 ++-- ...AddVotingConfigExclusionsRequestTests.java | 16 ++++----- ...tAddVotingConfigExclusionsActionTests.java | 22 ++++++++---- .../coordination/CoordinatorTests.java | 15 ++++---- .../cluster/coordination/NodeJoinTests.java | 3 +- ...stAddVotingConfigExclusionActionTests.java | 1 + .../test/test/InternalTestClusterTests.java | 4 +++ 11 files changed, 83 insertions(+), 62 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index f78d4b87f8886..65fbb1e88f771 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.action.admin.cluster.configuration; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -31,6 +32,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.logging.DeprecationLogger; import java.io.IOException; import java.util.Arrays; @@ -45,6 +47,9 @@ * configuration. */ public class AddVotingConfigExclusionsRequest extends MasterNodeRequest { + public static final String DEPRECATION_MESSAGE = "nodeDescription is deprecated and will be removed, use nodeIds or nodeNames instead"; + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(AddVotingConfigExclusionsRequest.class)); private final String[] nodeDescriptions; private final String[] nodeIds; private final String[] nodeNames; @@ -75,6 +80,10 @@ public AddVotingConfigExclusionsRequest(String[] nodeDescriptions, String[] node "One and only one of [node_name], [node_names] and [node_ids] has to be set"); } + if (nodeDescriptions.length > 0) { + deprecationLogger.deprecatedAndMaybeLog("voting_config_exclusion", DEPRECATION_MESSAGE); + } + this.nodeDescriptions = nodeDescriptions; this.nodeIds = nodeIds; this.nodeNames = nodeNames; @@ -83,7 +92,6 @@ public AddVotingConfigExclusionsRequest(String[] nodeDescriptions, String[] node public AddVotingConfigExclusionsRequest(StreamInput in) throws IOException { super(in); - // TODO should this be removed in the latest version where nodeIds and nodeNames are used? nodeDescriptions = in.readStringArray(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { nodeIds = in.readStringArray(); @@ -94,6 +102,12 @@ public AddVotingConfigExclusionsRequest(StreamInput in) throws IOException { nodeNames = Strings.EMPTY_ARRAY; } timeout = in.readTimeValue(); + + if (nodeDescriptions.length > 0) { + deprecationLogger.deprecatedAndMaybeLog("voting_config_exclusion", + "nodeDescription is deprecated and will be removed, use nodeIds or nodeNames instead"); + } + } Set resolveVotingConfigExclusions(ClusterState currentState) { @@ -177,25 +191,17 @@ Set resolveVotingConfigExclusionsAndCheckMaximum(ClusterS } private boolean noneOrMoreThanOneIsSet(String[] deprecatedNodeDescription, String[] nodeIds, String[] nodeNames) { - if(arrayHasElement(deprecatedNodeDescription)) { - return arrayHasElement(nodeIds) || arrayHasElement(nodeNames); - } - else if (arrayHasElement(nodeIds)) { - return arrayHasElement(nodeNames); + if(deprecatedNodeDescription.length > 0) { + return nodeIds.length > 0 || nodeNames.length > 0; } - else if (arrayHasElement(nodeNames)) { - return false; + else if (nodeIds.length > 0) { + return nodeNames.length > 0; } else { - // none of the node identifiers are set - return true; + return nodeNames.length > 0 == false; } } - private boolean arrayHasElement(String[] array) { - return array != null && array.length > 0; - } - /** * @return descriptions of the nodes for whom to add voting config exclusions. */ @@ -232,7 +238,6 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - // TODO should this be removed in the latest version where nodeIds and nodeNames are used? out.writeStringArray(nodeDescriptions); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeStringArray(nodeIds); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index eadb37c0f2d48..7b971dcadbdcc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -868,8 +868,7 @@ assert localNodeMayWinElection(getLastAcceptedState()) : // Package-private for testing ClusterState improveConfiguration(ClusterState clusterState) { assert Thread.holdsLock(mutex) : "Coordinator mutex not held"; - assert validVotingConfigExclusionState(clusterState) : "Voting Config Exclusion in invalid state. " + - "Exclusions may not be processed correctly"; + assert validVotingConfigExclusionState(clusterState) : clusterState; // exclude any nodes whose ID is in the voting config exclusions list ... final Stream excludedNodeIds = clusterState.getVotingConfigExclusions().stream().map(VotingConfigExclusion::getNodeId); @@ -913,9 +912,9 @@ private boolean validVotingConfigExclusionState(ClusterState clusterState) { .map(VotingConfigExclusion::getNodeId) .collect(Collectors.toSet()); for (DiscoveryNode node : clusterState.getNodes()) { - // TODO should this check be applied to master-eligible nodes only? - if(nodeIdsWithAbsentName.contains(node.getId()) || nodeNamesWithAbsentId.contains(node.getName())) { - return false; + if (node.isMasterNode() && + (nodeIdsWithAbsentName.contains(node.getId()) || nodeNamesWithAbsentId.contains(node.getName()))) { + return false; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java index 82bfd51ee4189..afd29adf4ab46 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.RerouteService; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import java.util.ArrayList; @@ -148,9 +149,9 @@ public ClusterTasksResult execute(ClusterState currentState, List jo nodesChanged = true; minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion()); maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion()); - // TODO do we need to check node.isMasterNode here for eligibility? I think since in the checks later master - // eligibility will be taken into account anyway, here we may not need the check? - joiniedNodeNameIds.put(node.getName(), node.getId()); + if(node.isMasterNode()) { + joiniedNodeNameIds.put(node.getName(), node.getId()); + } } catch (IllegalArgumentException | IllegalStateException e) { results.failure(joinTask, e); continue; @@ -177,14 +178,19 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } }).collect(Collectors.toSet()); - CoordinationMetaData.Builder coordMetaDataBuilder = CoordinationMetaData.builder(currentState.coordinationMetaData()) - .clearVotingConfigExclusions(); - newVotingConfigExclusions.forEach(coordMetaDataBuilder::addVotingConfigExclusion); - MetaData newMetaData = MetaData.builder(currentState.metaData()).coordinationMetaData(coordMetaDataBuilder.build()).build(); - return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).metaData(newMetaData).build())); - } else { - return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).build())); + // if VotingConfigExclusions did get updated + if (Sets.difference(currentVotingConfigExclusions, newVotingConfigExclusions).isEmpty() == false) { + CoordinationMetaData.Builder coordMetaDataBuilder = CoordinationMetaData.builder(currentState.coordinationMetaData()) + .clearVotingConfigExclusions(); + newVotingConfigExclusions.forEach(coordMetaDataBuilder::addVotingConfigExclusion); + MetaData newMetaData = MetaData.builder(currentState.metaData()) + .coordinationMetaData(coordMetaDataBuilder.build()).build(); + return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder) + .metaData(newMetaData).build())); + } } + + return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).build())); } else { // we must return a new cluster state instance to force publishing. This is important // for the joining node to finalize its join and set us as a master diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 75d633097314f..6f8f680b10afd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -337,7 +337,6 @@ public String[] resolveNodes(String... nodes) { return StreamSupport.stream(this.spliterator(), false).map(DiscoveryNode::getId).toArray(String[]::new); } else { ObjectHashSet resolvedNodesIds = new ObjectHashSet<>(nodes.length); - for (String nodeId : nodes) { if (nodeId.equals("_local")) { String localNodeId = getLocalNodeId(); @@ -354,9 +353,9 @@ public String[] resolveNodes(String... nodes) { } else { for (DiscoveryNode node : this) { if ("_all".equals(nodeId) - || Regex.simpleMatch(nodeId, node.getName()) - || Regex.simpleMatch(nodeId, node.getHostAddress()) - || Regex.simpleMatch(nodeId, node.getHostName())) { + || Regex.simpleMatch(nodeId, node.getName()) + || Regex.simpleMatch(nodeId, node.getHostAddress()) + || Regex.simpleMatch(nodeId, node.getHostName())) { resolvedNodesIds.add(node.getId()); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java index fd25844ee11e9..01355c5e50e71 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionAction.java @@ -39,8 +39,11 @@ public class RestAddVotingConfigExclusionAction extends BaseRestHandler { private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(30L); private static final Logger logger = LogManager.getLogger(RestAddVotingConfigExclusionAction.class); - private static final String DEPRECATION_MESSAGE = "Using [node_name] for adding voting config exclustion will be removed " + - "in a future version. Please use [node_ids] or [node_names] instead"; + private static final String DEPRECATION_MESSAGE = "POST /_cluster/voting_config_exclusions/{node_name} " + + "will be removed in a future version. " + + "Please use POST /_cluster/voting_config_exclusions?node_ids=... " + + "or POST /_cluster/voting_config_exclusions?node_names=... instead."; + @Override public String getName() { return "add_voting_config_exclusions_action"; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index 119757064aad2..495550796f9d9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -54,10 +54,10 @@ public void testSerialization() throws IOException { AddVotingConfigExclusionsRequest::new); assertThat(deserialized.getNodeDescriptions(), equalTo(originalRequest.getNodeDescriptions())); assertThat(deserialized.getTimeout(), equalTo(originalRequest.getTimeout())); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testSerializationForNodeIdOrNodeName() throws IOException { - // TODO still need adjustment for version? copyWriteable uses Version.CURRENT AddVotingConfigExclusionsRequest originalRequest = new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, new String[]{"nodeId1", "nodeId2"}, Strings.EMPTY_ARRAY, TimeValue.ZERO); AddVotingConfigExclusionsRequest deserialized = copyWriteable(originalRequest, writableRegistry(), @@ -68,7 +68,6 @@ public void testSerializationForNodeIdOrNodeName() throws IOException { assertThat(deserialized.getNodeNames(), equalTo(originalRequest.getNodeNames())); assertThat(deserialized.getTimeout(), equalTo(originalRequest.getTimeout())); - // TODO still need adjustment for version? copyWriteable uses Version.CURRENT originalRequest = new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, new String[]{"nodeName1", "nodeName2"}, TimeValue.ZERO); deserialized = copyWriteable(originalRequest, writableRegistry(), @@ -112,23 +111,19 @@ public void testResolve() { .add(localNode).add(otherNode1).add(otherNode2).add(otherDataNode).localNodeId(localNode.getId())).build(); assertThat(makeRequest("_all").resolveVotingConfigExclusions(clusterState), - containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); + containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); assertThat(makeRequest("_local").resolveVotingConfigExclusions(clusterState), - contains(localNodeExclusion)); + contains(localNodeExclusion)); assertThat(makeRequest("other*").resolveVotingConfigExclusions(clusterState), - containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); assertThat(expectThrows(IllegalArgumentException.class, () -> makeRequest("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testResolveAllNodeIdentifiersNullOrEmpty() { - assertThat(expectThrows(IllegalArgumentException.class, - () -> new AddVotingConfigExclusionsRequest(null, null, null, TimeValue.ZERO)).getMessage(), - equalTo("Please set node identifiers correctly. " + - "One and only one of [node_name], [node_names] and [node_ids] has to be set")); - assertThat(expectThrows(IllegalArgumentException.class, () -> new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.ZERO)) .getMessage(), equalTo("Please set node identifiers correctly. " + @@ -328,6 +323,7 @@ public void testResolveAndCheckMaximum() { () -> makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")).getMessage(), equalTo("add voting config exclusions request for [_local] would add [1] exclusions to the existing [1] which would " + "exceed the maximum of [1] set by [setting.name]")); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } private static AddVotingConfigExclusionsRequest makeRequest(String... descriptions) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 09a16a1b5d1d2..253c682852b61 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -149,6 +149,7 @@ public void testWithdrawsVoteFromANode() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { @@ -166,6 +167,7 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedException { @@ -182,7 +184,8 @@ public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedExce assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), - containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromAllMasterEligibleNodes() throws InterruptedException { @@ -199,7 +202,8 @@ public void testWithdrawsVotesFromAllMasterEligibleNodes() throws InterruptedExc assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), - containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); + containsInAnyOrder(localNodeExclusion, otherNode1Exclusion, otherNode2Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVoteFromLocalNode() throws InterruptedException { @@ -216,7 +220,8 @@ public void testWithdrawsVoteFromLocalNode() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), - contains(localNodeExclusion)); + contains(localNodeExclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedException { final ClusterState state = clusterService.state(); @@ -231,7 +236,7 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc // no observer to reconfigure transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.ZERO), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -241,6 +246,7 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testReturnsErrorIfNoMatchingNodes() throws InterruptedException { @@ -260,13 +266,13 @@ public void testReturnsErrorIfNoMatchingNodes() throws InterruptedException { assertThat(rootCause, instanceOf(IllegalArgumentException.class)); assertThat(rootCause.getMessage(), equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testOnlyMatchesMasterEligibleNodes() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); final SetOnce exceptionHolder = new SetOnce<>(); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, new AddVotingConfigExclusionsRequest(new String[]{"_all", "master:false"}), expectError(e -> { @@ -275,12 +281,12 @@ public void testOnlyMatchesMasterEligibleNodes() throws InterruptedException { }) ); - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); final Throwable rootCause = exceptionHolder.get().getRootCause(); assertThat(rootCause, instanceOf(IllegalArgumentException.class)); assertThat(rootCause.getMessage(), equalTo("add voting config exclusions request for [_all, master:false] matched no master-eligible nodes")); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testExcludeAbsentNodesByNodeIds() throws InterruptedException { @@ -376,6 +382,7 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testExcludeByNodeIdSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedException { @@ -470,6 +477,7 @@ public void testReturnsErrorIfMaximumExclusionCountExceeded() throws Interrupted assertThat(rootCause.getMessage(), equalTo("add voting config exclusions request for [other*] would add [" + newCount + "] exclusions to the existing [" + existingCount + "] which would exceed the maximum of [2] set by [cluster.max_voting_config_exclusions]")); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testTimesOut() throws InterruptedException { @@ -489,6 +497,8 @@ public void testTimesOut() throws InterruptedException { final Throwable rootCause = exceptionHolder.get().getRootCause(); assertThat(rootCause,instanceOf(ElasticsearchTimeoutException.class)); assertThat(rootCause.getMessage(), startsWith("timed out waiting for voting config exclusions [{other1}")); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); + } private TransportResponseHandler expectSuccess( diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index d71808a365670..282c2b2a49bb0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.coordination.Coordinator.Mode; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.Loggers; @@ -58,7 +59,6 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; import static org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.DEFAULT_DELAY_VARIABILITY; import static org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.EXTREME_DELAY_VARIABILITY; import static org.elasticsearch.cluster.coordination.Coordinator.Mode.CANDIDATE; @@ -1437,9 +1437,7 @@ public void testImprovecOnfigurationPerformsVotingConfigExclusionStateCheck() { ClusterState newState1 = buildNewClusterStateWithVotingConfigExclusion(currentState, newVotingConfigExclusion1); synchronized (coordinator.mutex) { - AssertionError error = expectThrows(AssertionError.class, () -> coordinator.improveConfiguration(newState1)); - assertEquals("Voting Config Exclusion in invalid state. Exclusions may not be processed correctly", - error.getMessage()); + expectThrows(AssertionError.class, () -> coordinator.improveConfiguration(newState1)); } Set newVotingConfigExclusion2 = new HashSet<>(){{ @@ -1447,12 +1445,10 @@ public void testImprovecOnfigurationPerformsVotingConfigExclusionStateCheck() { "resolvableNodeName")); }}; - ClusterState newState2 = buildNewClusterStateWithVotingConfigExclusion(currentState, newVotingConfigExclusion1); + ClusterState newState2 = buildNewClusterStateWithVotingConfigExclusion(currentState, newVotingConfigExclusion2); synchronized (coordinator.mutex) { - AssertionError error = expectThrows(AssertionError.class, () -> coordinator.improveConfiguration(newState2)); - assertEquals("Voting Config Exclusion in invalid state. Exclusions may not be processed correctly", - error.getMessage()); + expectThrows(AssertionError.class, () -> coordinator.improveConfiguration(newState2)); } } } @@ -1461,7 +1457,8 @@ private ClusterState buildNewClusterStateWithVotingConfigExclusion(ClusterState Set newVotingConfigExclusion) { DiscoveryNodes newNodes = DiscoveryNodes.builder(currentState.nodes()) .add(new DiscoveryNode("resolvableNodeName", "resolvableNodeId", buildNewFakeTransportAddress(), - emptyMap(), emptySet(), Version.CURRENT)).build(); + emptyMap(), Set.of(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT)) + .build(); CoordinationMetaData.Builder coordMetaDataBuilder = CoordinationMetaData.builder(currentState.coordinationMetaData()); newVotingConfigExclusion.forEach(coordMetaDataBuilder::addVotingConfigExclusion); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java index ab1cc54f96fa9..f6bd8f528b090 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java @@ -304,7 +304,8 @@ public void testJoinWithHigherTermButBetterStateGetsRejected() { assertFalse(isLocalNodeElectedMaster()); } - public void testJoinWithHigherTermButBetterStateStillElectsMasterThroughSelfJoin() { DiscoveryNode node0 = newNode(0, true); + public void testJoinWithHigherTermButBetterStateStillElectsMasterThroughSelfJoin() { + DiscoveryNode node0 = newNode(0, true); DiscoveryNode node1 = newNode(1, true); long initialTerm = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java index 2595f5d652df1..eb712429e1403 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestAddVotingConfigExclusionActionTests.java @@ -53,6 +53,7 @@ public void testResolveVotingConfigExclusionsRequest() { assertArrayEquals(expected, addVotingConfigExclusionsRequest.getNodeDescriptions()); assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeIds()); assertArrayEquals(Strings.EMPTY_ARRAY, addVotingConfigExclusionsRequest.getNodeNames()); + assertWarnings("nodeDescription is deprecated and will be removed, use nodeIds or nodeNames instead"); } public void testResolveVotingConfigExclusionsRequestNodeIds() { diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index f7e0aa526290a..7daec8d7ab30e 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.test; import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; @@ -280,6 +281,8 @@ public Path nodeConfigPath(int nodeOrdinal) { } finally { cluster.close(); } + + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } private Path[] getNodePaths(InternalTestCluster cluster, String name) { @@ -408,6 +411,7 @@ public Path nodeConfigPath(int nodeOrdinal) { } } finally { cluster.close(); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } } From 0fb29c3e76656cd13b4c8ee6e523ceb2adff97cc Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 25 Mar 2020 22:16:04 -0700 Subject: [PATCH 14/29] Make Coordinator#validVotingConfigExclusionState package static for testability --- .../elasticsearch/cluster/coordination/Coordinator.java | 2 +- .../cluster/coordination/CoordinatorTests.java | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 7b971dcadbdcc..f1b63f2814c19 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -901,7 +901,7 @@ ClusterState improveConfiguration(ClusterState clusterState) { * 1. Every voting config exclusion with an ID of _absent_ should not match any nodes currently in the cluster by name * 2. Every voting config exclusion with a name of _absent_ should not match any nodes currently in the cluster by ID */ - private boolean validVotingConfigExclusionState(ClusterState clusterState) { + static boolean validVotingConfigExclusionState(ClusterState clusterState) { Set votingConfigExclusions = clusterState.getVotingConfigExclusions(); Set nodeNamesWithAbsentId = votingConfigExclusions.stream() .filter(e -> e.getNodeId().equals(VotingConfigExclusion.MISSING_VALUE_MARKER)) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 282c2b2a49bb0..6c7dbac1d5608 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -1436,9 +1436,7 @@ public void testImprovecOnfigurationPerformsVotingConfigExclusionStateCheck() { ClusterState newState1 = buildNewClusterStateWithVotingConfigExclusion(currentState, newVotingConfigExclusion1); - synchronized (coordinator.mutex) { - expectThrows(AssertionError.class, () -> coordinator.improveConfiguration(newState1)); - } + assertFalse(Coordinator.validVotingConfigExclusionState(newState1)); Set newVotingConfigExclusion2 = new HashSet<>(){{ add(new CoordinationMetaData.VotingConfigExclusion(CoordinationMetaData.VotingConfigExclusion.MISSING_VALUE_MARKER, @@ -1447,9 +1445,7 @@ public void testImprovecOnfigurationPerformsVotingConfigExclusionStateCheck() { ClusterState newState2 = buildNewClusterStateWithVotingConfigExclusion(currentState, newVotingConfigExclusion2); - synchronized (coordinator.mutex) { - expectThrows(AssertionError.class, () -> coordinator.improveConfiguration(newState2)); - } + assertFalse(Coordinator.validVotingConfigExclusionState(newState2)); } } From aaa0f895ff2fa09ac0922997314944946647d042 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 26 Mar 2020 18:25:06 -0700 Subject: [PATCH 15/29] Refactoring for node resolution logic and NodeJoinTest --- .../AddVotingConfigExclusionsRequest.java | 75 +++++++------------ ...AddVotingConfigExclusionsRequestTests.java | 3 +- .../cluster/coordination/NodeJoinTests.java | 22 +++++- 3 files changed, 46 insertions(+), 54 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 65fbb1e88f771..6d9a58c00c0a6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -30,8 +30,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.iterable.Iterables; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.logging.DeprecationLogger; import java.io.IOException; @@ -41,6 +39,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; /** * A request to add voting config exclusions for certain master-eligible nodes, and wait for these nodes to be removed from the voting @@ -112,67 +111,47 @@ public AddVotingConfigExclusionsRequest(StreamInput in) throws IOException { Set resolveVotingConfigExclusions(ClusterState currentState) { final DiscoveryNodes allNodes = currentState.nodes(); - Set allProcessedNodes = null; + Set newVotingConfigExclusions = new HashSet<>(); if (nodeDescriptions.length >= 1) { - allProcessedNodes = Arrays.stream(allNodes.resolveNodes(nodeDescriptions)).map(allNodes::get) - .filter(DiscoveryNode::isMasterNode).map(VotingConfigExclusion::new).collect(Collectors.toSet()); + newVotingConfigExclusions = Arrays.stream(allNodes.resolveNodes(nodeDescriptions)).map(allNodes::get) + .filter(DiscoveryNode::isMasterNode).map(VotingConfigExclusion::new).collect(Collectors.toSet()); - if (allProcessedNodes.isEmpty()) { + if (newVotingConfigExclusions.isEmpty()) { throw new IllegalArgumentException("add voting config exclusions request for " + Arrays.asList(nodeDescriptions) + " matched no master-eligible nodes"); } - } else { - Set resolvedVotingConfigExclusions; - Set unresolvedVotingConfigExclusions; - - Set resolvedNodes = new HashSet<>(nodeIds.length); - Set unresolvedNodes = new HashSet<>(); - - if (nodeIds.length >= 1) { - for (String nodeId : nodeIds) { - if (allNodes.nodeExists(nodeId)) { - resolvedNodes.add(nodeId); - } - else { - unresolvedNodes.add(nodeId); + } else if (nodeIds.length >= 1) { + for (String nodeId : nodeIds) { + if (allNodes.nodeExists(nodeId)) { + DiscoveryNode discoveryNode = allNodes.get(nodeId); + if (discoveryNode.isMasterNode()) { + newVotingConfigExclusions.add(new VotingConfigExclusion(discoveryNode)); } } - - unresolvedVotingConfigExclusions = unresolvedNodes.stream() - .map(nodeId -> new VotingConfigExclusion(nodeId, VotingConfigExclusion.MISSING_VALUE_MARKER)) - .collect(Collectors.toSet()); + else { + newVotingConfigExclusions.add(new VotingConfigExclusion(nodeId, VotingConfigExclusion.MISSING_VALUE_MARKER)); + } } - else { - Map existingNodesNameId = new HashMap<>(); - for (DiscoveryNode node : allNodes) { - if (node.isMasterNode()) { - existingNodesNameId.put(node.getName(), node.getId()); + } else { + Map existingNodeNameId = StreamSupport.stream(allNodes.spliterator(), false) + .collect(Collectors.toMap(DiscoveryNode::getName, DiscoveryNode::getId)); + + for (String nodeName : nodeNames) { + if (existingNodeNameId.containsKey(nodeName)){ + DiscoveryNode discoveryNode = allNodes.get(existingNodeNameId.get(nodeName)); + if (discoveryNode.isMasterNode()) { + newVotingConfigExclusions.add(new VotingConfigExclusion(discoveryNode)); } } - - for (String nodeName : nodeNames) { - if (existingNodesNameId.containsKey(nodeName)){ - resolvedNodes.add(existingNodesNameId.get(nodeName)); - } - else { - unresolvedNodes.add(nodeName); - } + else { + newVotingConfigExclusions.add(new VotingConfigExclusion(VotingConfigExclusion.MISSING_VALUE_MARKER, nodeName)); } - - unresolvedVotingConfigExclusions = unresolvedNodes.stream() - .map(nodeName -> new VotingConfigExclusion(VotingConfigExclusion.MISSING_VALUE_MARKER, nodeName)) - .collect(Collectors.toSet()); } - - resolvedVotingConfigExclusions = resolvedNodes.stream() - .map(allNodes::get).filter(DiscoveryNode::isMasterNode).map(VotingConfigExclusion::new).collect(Collectors.toSet()); - - allProcessedNodes = Sets.newHashSet(Iterables.concat(resolvedVotingConfigExclusions, unresolvedVotingConfigExclusions)); } - allProcessedNodes.removeIf(n -> currentState.getVotingConfigExclusions().contains(n)); - return allProcessedNodes; + newVotingConfigExclusions.removeIf(n -> currentState.getVotingConfigExclusions().contains(n)); + return newVotingConfigExclusions; } Set resolveVotingConfigExclusionsAndCheckMaximum(ClusterState currentState, int maxExclusionsCount, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index 495550796f9d9..347f40f921e1a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -197,7 +197,6 @@ public void testResolveByNodeIds() { containsInAnyOrder(node1Exclusion, unresolvableVotingConfigExclusion)); } - public void testResolveByNodeNames() { final DiscoveryNode node1 = new DiscoveryNode( "nodeName1", @@ -318,7 +317,7 @@ public void testResolveAndCheckMaximum() { final ClusterState clusterState = builder.build(); assertThat(makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), - contains(localNodeExclusion)); + contains(localNodeExclusion)); assertThat(expectThrows(IllegalArgumentException.class, () -> makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")).getMessage(), equalTo("add voting config exclusions request for [_local] would add [1] exclusions to the existing [1] which would " + diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java index f6bd8f528b090..bd3c7ca8f95ec 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java @@ -113,8 +113,6 @@ private static ClusterState initialState(DiscoveryNode localNode, long term, lon .term(term) .lastAcceptedConfiguration(config) .lastCommittedConfiguration(config) - .addVotingConfigExclusion(new CoordinationMetaData.VotingConfigExclusion( - CoordinationMetaData.VotingConfigExclusion.MISSING_VALUE_MARKER, "knownNodeName")) .build())) .version(version) .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build(); @@ -384,8 +382,11 @@ public void testJoinUpdateVotingConfigExclusion() throws Exception { long initialTerm = randomLongBetween(1, 10); long initialVersion = randomLongBetween(1, 10); - setupFakeMasterServiceAndCoordinator(initialTerm, initialState(initialNode, initialTerm, initialVersion, - new VotingConfiguration(Collections.singleton(initialNode.getId())))); + CoordinationMetaData.VotingConfigExclusion votingConfigExclusion = new CoordinationMetaData.VotingConfigExclusion( + CoordinationMetaData.VotingConfigExclusion.MISSING_VALUE_MARKER, "knownNodeName"); + + setupFakeMasterServiceAndCoordinator(initialTerm, buildStateWithVotingConfigExclusion(initialNode, initialTerm, + initialVersion, votingConfigExclusion)); DiscoveryNode knownJoiningNode = new DiscoveryNode("knownNodeName", "newNodeId", buildNewFakeTransportAddress(), emptyMap(), Set.of(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); @@ -400,6 +401,19 @@ public void testJoinUpdateVotingConfigExclusion() throws Exception { })); } + private ClusterState buildStateWithVotingConfigExclusion(DiscoveryNode initialNode, + long initialTerm, + long initialVersion, + CoordinationMetaData.VotingConfigExclusion votingConfigExclusion) { + ClusterState initialState = initialState(initialNode, initialTerm, initialVersion, + new VotingConfiguration(Collections.singleton(initialNode.getId()))); + MetaData newMetaData = MetaData.builder(initialState.metaData()) + .coordinationMetaData(CoordinationMetaData.builder(initialState.coordinationMetaData()) + .addVotingConfigExclusion(votingConfigExclusion).build()).build(); + + return ClusterState.builder(initialState).metaData(newMetaData).build(); + } + private void handleStartJoinFrom(DiscoveryNode node, long term) throws Exception { final RequestHandlerRegistry startJoinHandler = (RequestHandlerRegistry) transport.getRequestHandler(JoinHelper.START_JOIN_ACTION_NAME); From 02a353317d415560ba94947771f09e7fa1257277 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 26 Mar 2020 19:28:55 -0700 Subject: [PATCH 16/29] Refactor out deprecated AddVotingConfigExclusionsRequest constructor --- .../AddVotingConfigExclusionsRequest.java | 9 ----- ...AddVotingConfigExclusionsRequestTests.java | 17 ++++----- ...tAddVotingConfigExclusionsActionTests.java | 35 +++++++++++-------- .../cluster/MinimumMasterNodesIT.java | 8 +++-- .../cluster/SpecificMasterNodesIT.java | 5 ++- .../coordination/VotingConfigurationIT.java | 5 ++- .../gateway/RecoveryFromGatewayIT.java | 4 ++- .../test/InternalTestCluster.java | 3 +- 8 files changed, 48 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 6d9a58c00c0a6..2324b460def5b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -54,15 +54,6 @@ public class AddVotingConfigExclusionsRequest extends MasterNodeRequest makeRequest("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), + () -> makeRequestWithNodeDescriptions("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } @@ -316,16 +316,17 @@ public void testResolveAndCheckMaximum() { .coordinationMetaData(CoordinationMetaData.builder().addVotingConfigExclusion(otherNode1Exclusion).build())); final ClusterState clusterState = builder.build(); - assertThat(makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), + assertThat(makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), contains(localNodeExclusion)); assertThat(expectThrows(IllegalArgumentException.class, - () -> makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")).getMessage(), + () -> makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")) + .getMessage(), equalTo("add voting config exclusions request for [_local] would add [1] exclusions to the existing [1] which would " + "exceed the maximum of [1] set by [setting.name]")); assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } - private static AddVotingConfigExclusionsRequest makeRequest(String... descriptions) { - return new AddVotingConfigExclusionsRequest(descriptions); + private static AddVotingConfigExclusionsRequest makeRequestWithNodeDescriptions(String... descriptions) { + return new AddVotingConfigExclusionsRequest(descriptions, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 253c682852b61..b79065ebec112 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -140,7 +140,8 @@ public void testWithdrawsVoteFromANode() throws InterruptedException { clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}), + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{"other1"}, TimeValue.timeValueSeconds(30)), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -149,7 +150,6 @@ public void testWithdrawsVoteFromANode() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { @@ -157,7 +157,8 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1", "other2"}), + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{"other1", "other2"}, TimeValue.timeValueSeconds(30)), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -167,7 +168,6 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedException { @@ -175,7 +175,8 @@ public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedExce clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other*"}), + new AddVotingConfigExclusionsRequest(new String[]{"other*"}, Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -193,7 +194,8 @@ public void testWithdrawsVotesFromAllMasterEligibleNodes() throws InterruptedExc clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_all"}), + new AddVotingConfigExclusionsRequest(new String[]{"_all"}, Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -211,7 +213,8 @@ public void testWithdrawsVoteFromLocalNode() throws InterruptedException { clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_local"}), + new AddVotingConfigExclusionsRequest(new String[]{"_local"}, Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -236,7 +239,7 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc // no observer to reconfigure transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.ZERO), + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, new String[]{"other1"}, TimeValue.ZERO), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -246,15 +249,15 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } - public void testReturnsErrorIfNoMatchingNodes() throws InterruptedException { + public void testReturnsErrorIfNoMatchingNodesWithDeprecatedNodeDescriptions() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"not-a-node"}), + new AddVotingConfigExclusionsRequest(new String[]{"not-a-node"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + TimeValue.timeValueSeconds(30)), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); @@ -274,7 +277,8 @@ public void testOnlyMatchesMasterEligibleNodes() throws InterruptedException { final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_all", "master:false"}), + new AddVotingConfigExclusionsRequest(new String[]{"_all", "master:false"}, Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); @@ -372,7 +376,8 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce final CountDownLatch countDownLatch = new CountDownLatch(1); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}), + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{"other1"}, TimeValue.timeValueSeconds(30)), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -382,7 +387,6 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testExcludeByNodeIdSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedException { @@ -464,7 +468,8 @@ public void testReturnsErrorIfMaximumExclusionCountExceeded() throws Interrupted final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other*"}), + new AddVotingConfigExclusionsRequest(new String[]{"other*"}, Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); diff --git a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index e9043528f9c0c..2e5708b31b827 100644 --- a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -31,7 +31,9 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; @@ -123,7 +125,8 @@ public void testTwoNodesNoMasterBlock() throws Exception { String masterNode = internalCluster().getMasterName(); String otherNode = node1Name.equals(masterNode) ? node2Name : node1Name; logger.info("--> add voting config exclusion for non-master node, to be sure it's not elected"); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{otherNode})).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, new String[]{otherNode}, TimeValue.timeValueSeconds(30))).get(); logger.info("--> stop master node, no master block should appear"); Settings masterDataPathSettings = internalCluster().dataPathSettings(masterNode); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNode)); @@ -170,7 +173,8 @@ public void testTwoNodesNoMasterBlock() throws Exception { masterNode = internalCluster().getMasterName(); otherNode = node1Name.equals(masterNode) ? node2Name : node1Name; logger.info("--> add voting config exclusion for master node, to be sure it's not elected"); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{masterNode})).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, new String[]{masterNode}, TimeValue.timeValueSeconds(30))).get(); logger.info("--> stop non-master node, no master block should appear"); Settings otherNodeDataPathSettings = internalCluster().dataPathSettings(otherNode); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(otherNode)); diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index 0e8f0f3f7e193..e47ca36a71f23 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -22,7 +22,9 @@ import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; @@ -115,7 +117,8 @@ public void testElectOnlyBetweenMasterNodes() throws Exception { logger.info("--> closing master node (1)"); client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(new String[]{masterNodeName})).get(); + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, new String[]{masterNodeName}, + TimeValue.timeValueSeconds(30))).get(); // removing the master from the voting configuration immediately triggers the master to step down assertBusy(() -> { assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState() diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java index c3c3abf8866b7..65a8e79b179a2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java @@ -24,6 +24,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; @@ -55,7 +57,8 @@ public void testAbdicateAfterVotingConfigExclusionAdded() throws ExecutionExcept logger.info("--> excluding master node {}", originalMaster); client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(new String[]{originalMaster})).get(); + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + new String[]{originalMaster}, TimeValue.timeValueSeconds(30))).get(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get(); assertNotEquals(originalMaster, internalCluster().getMasterName()); } diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index f20bee58d0487..017fa5d291ff3 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; @@ -309,7 +310,8 @@ public void testTwoNodeFirstNodeCleared() throws Exception { Map primaryTerms = assertAndCapturePrimaryTerms(null); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{firstNode})).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, new String[]{firstNode}, TimeValue.timeValueSeconds(30))).get(); internalCluster().fullRestart(new RestartCallback() { @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index fef1dcacba309..f664574fbaa3a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1673,7 +1673,8 @@ private Set excludeMasters(Collection nodeAndClients) { logger.info("adding voting config exclusions {} prior to restart/shutdown", excludedNodeIds); try { client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(excludedNodeIds.toArray(Strings.EMPTY_ARRAY))).get(); + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, excludedNodeIds.toArray(Strings.EMPTY_ARRAY), + Strings.EMPTY_ARRAY, timeValueSeconds(30))).get(); } catch (InterruptedException | ExecutionException e) { throw new AssertionError("unexpected", e); } From 53f133c3c434d5aa22a4f79a2e4fdc3badae63a3 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 26 Mar 2020 23:53:24 -0700 Subject: [PATCH 17/29] Fix checkstyle and tests --- .../configuration/AddVotingConfigExclusionsRequest.java | 1 - .../org/elasticsearch/test/test/InternalTestClusterTests.java | 4 ---- 2 files changed, 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 2324b460def5b..672bcdb788d71 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 7daec8d7ab30e..f7e0aa526290a 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.test.test; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; @@ -281,8 +280,6 @@ public Path nodeConfigPath(int nodeOrdinal) { } finally { cluster.close(); } - - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } private Path[] getNodePaths(InternalTestCluster cluster, String name) { @@ -411,7 +408,6 @@ public Path nodeConfigPath(int nodeOrdinal) { } } finally { cluster.close(); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } } From 0f6dd6fd5e9d1501c84345f1c666659fed1bd9ef Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Fri, 27 Mar 2020 00:54:14 -0700 Subject: [PATCH 18/29] Fix test failure due to misnomer --- .../elasticsearch/test/InternalTestCluster.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 5f40c9f7a3e7d..144a2bd21e3e0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1655,7 +1655,7 @@ private NodeAndClient removeNode(NodeAndClient nodeAndClient) { private Set excludeMasters(Collection nodeAndClients) { assert Thread.holdsLock(this); - final Set excludedNodeIds = new HashSet<>(); + final Set excludedNodeNames = new HashSet<>(); if (autoManageMasterNodes && nodeAndClients.size() > 0) { final long currentMasters = nodes.values().stream().filter(NodeAndClient::isMasterEligible).count(); @@ -1667,20 +1667,20 @@ private Set excludeMasters(Collection nodeAndClients) { // However, we do not yet have a way to be sure there's a majority left, because the voting configuration may not yet have // been updated when the previous nodes shut down, so we must always explicitly withdraw votes. // TODO add cluster health API to check that voting configuration is optimal so this isn't always needed - nodeAndClients.stream().filter(NodeAndClient::isMasterEligible).map(NodeAndClient::getName).forEach(excludedNodeIds::add); - assert excludedNodeIds.size() == stoppingMasters; + nodeAndClients.stream().filter(NodeAndClient::isMasterEligible).map(NodeAndClient::getName).forEach(excludedNodeNames::add); + assert excludedNodeNames.size() == stoppingMasters; - logger.info("adding voting config exclusions {} prior to restart/shutdown", excludedNodeIds); + logger.info("adding voting config exclusions {} prior to restart/shutdown", excludedNodeNames); try { client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, excludedNodeIds.toArray(Strings.EMPTY_ARRAY), - Strings.EMPTY_ARRAY, timeValueSeconds(30))).get(); + new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, + excludedNodeNames.toArray(Strings.EMPTY_ARRAY), timeValueSeconds(30))).get(); } catch (InterruptedException | ExecutionException e) { throw new AssertionError("unexpected", e); } } } - return excludedNodeIds; + return excludedNodeNames; } private void removeExclusions(Set excludedNodeIds) { From b0d0a976f6366dd7370239e42aa6858837380e26 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 8 Apr 2020 18:03:32 -0700 Subject: [PATCH 19/29] Revert "Refactor out deprecated AddVotingConfigExclusionsRequest constructor" This reverts commit 02a353317d415560ba94947771f09e7fa1257277. --- .../AddVotingConfigExclusionsRequest.java | 9 +++++ ...AddVotingConfigExclusionsRequestTests.java | 17 +++++---- ...tAddVotingConfigExclusionsActionTests.java | 35 ++++++++----------- .../cluster/MinimumMasterNodesIT.java | 8 ++--- .../cluster/SpecificMasterNodesIT.java | 5 +-- .../coordination/VotingConfigurationIT.java | 5 +-- .../gateway/RecoveryFromGatewayIT.java | 4 +-- .../test/InternalTestCluster.java | 3 +- 8 files changed, 38 insertions(+), 48 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 672bcdb788d71..606ace8544f60 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -53,6 +53,15 @@ public class AddVotingConfigExclusionsRequest extends MasterNodeRequest makeRequestWithNodeDescriptions("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), + () -> makeRequest("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } @@ -316,17 +316,16 @@ public void testResolveAndCheckMaximum() { .coordinationMetaData(CoordinationMetaData.builder().addVotingConfigExclusion(otherNode1Exclusion).build())); final ClusterState clusterState = builder.build(); - assertThat(makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), + assertThat(makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), contains(localNodeExclusion)); assertThat(expectThrows(IllegalArgumentException.class, - () -> makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")) - .getMessage(), + () -> makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")).getMessage(), equalTo("add voting config exclusions request for [_local] would add [1] exclusions to the existing [1] which would " + "exceed the maximum of [1] set by [setting.name]")); assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } - private static AddVotingConfigExclusionsRequest makeRequestWithNodeDescriptions(String... descriptions) { - return new AddVotingConfigExclusionsRequest(descriptions, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)); + private static AddVotingConfigExclusionsRequest makeRequest(String... descriptions) { + return new AddVotingConfigExclusionsRequest(descriptions); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 99ad0e618de7e..de5a35dea0fb2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -153,8 +153,7 @@ public void testWithdrawsVoteFromANode() throws InterruptedException { clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{"other1"}, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -163,6 +162,7 @@ public void testWithdrawsVoteFromANode() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { @@ -170,8 +170,7 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{"other1", "other2"}, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"other1", "other2"}), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -181,6 +180,7 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedException { @@ -188,8 +188,7 @@ public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedExce clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other*"}, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"other*"}), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -207,8 +206,7 @@ public void testWithdrawsVotesFromAllMasterEligibleNodes() throws InterruptedExc clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_all"}, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"_all"}), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -226,8 +224,7 @@ public void testWithdrawsVoteFromLocalNode() throws InterruptedException { clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_local"}, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"_local"}), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -252,7 +249,7 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc // no observer to reconfigure transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, new String[]{"other1"}, TimeValue.ZERO), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.ZERO), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -262,15 +259,15 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } - public void testReturnsErrorIfNoMatchingNodesWithDeprecatedNodeDescriptions() throws InterruptedException { + public void testReturnsErrorIfNoMatchingNodes() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"not-a-node"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"not-a-node"}), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); @@ -290,8 +287,7 @@ public void testOnlyMatchesMasterEligibleNodes() throws InterruptedException { final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_all", "master:false"}, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"_all", "master:false"}), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); @@ -389,8 +385,7 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce final CountDownLatch countDownLatch = new CountDownLatch(1); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{"other1"}, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"other1"}), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -400,6 +395,7 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); + assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testExcludeByNodeIdSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedException { @@ -493,8 +489,7 @@ public void testReturnsErrorIfMaximumExclusionCountExceeded() throws Interrupted final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other*"}, Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest(new String[]{"other*"}), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); diff --git a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index 2e5708b31b827..e9043528f9c0c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -31,9 +31,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; @@ -125,8 +123,7 @@ public void testTwoNodesNoMasterBlock() throws Exception { String masterNode = internalCluster().getMasterName(); String otherNode = node1Name.equals(masterNode) ? node2Name : node1Name; logger.info("--> add voting config exclusion for non-master node, to be sure it's not elected"); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, new String[]{otherNode}, TimeValue.timeValueSeconds(30))).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{otherNode})).get(); logger.info("--> stop master node, no master block should appear"); Settings masterDataPathSettings = internalCluster().dataPathSettings(masterNode); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNode)); @@ -173,8 +170,7 @@ public void testTwoNodesNoMasterBlock() throws Exception { masterNode = internalCluster().getMasterName(); otherNode = node1Name.equals(masterNode) ? node2Name : node1Name; logger.info("--> add voting config exclusion for master node, to be sure it's not elected"); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, new String[]{masterNode}, TimeValue.timeValueSeconds(30))).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{masterNode})).get(); logger.info("--> stop non-master node, no master block should appear"); Settings otherNodeDataPathSettings = internalCluster().dataPathSettings(otherNode); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(otherNode)); diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index e47ca36a71f23..0e8f0f3f7e193 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -22,9 +22,7 @@ import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; @@ -117,8 +115,7 @@ public void testElectOnlyBetweenMasterNodes() throws Exception { logger.info("--> closing master node (1)"); client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, new String[]{masterNodeName}, - TimeValue.timeValueSeconds(30))).get(); + new AddVotingConfigExclusionsRequest(new String[]{masterNodeName})).get(); // removing the master from the voting configuration immediately triggers the master to step down assertBusy(() -> { assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState() diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java index 65a8e79b179a2..c3c3abf8866b7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java @@ -24,8 +24,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; @@ -57,8 +55,7 @@ public void testAbdicateAfterVotingConfigExclusionAdded() throws ExecutionExcept logger.info("--> excluding master node {}", originalMaster); client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{originalMaster}, TimeValue.timeValueSeconds(30))).get(); + new AddVotingConfigExclusionsRequest(new String[]{originalMaster})).get(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get(); assertNotEquals(originalMaster, internalCluster().getMasterName()); } diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 017fa5d291ff3..f20bee58d0487 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; @@ -310,8 +309,7 @@ public void testTwoNodeFirstNodeCleared() throws Exception { Map primaryTerms = assertAndCapturePrimaryTerms(null); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, new String[]{firstNode}, TimeValue.timeValueSeconds(30))).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{firstNode})).get(); internalCluster().fullRestart(new RestartCallback() { @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 144a2bd21e3e0..0ef4c4a00cf34 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1673,8 +1673,7 @@ private Set excludeMasters(Collection nodeAndClients) { logger.info("adding voting config exclusions {} prior to restart/shutdown", excludedNodeNames); try { client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - excludedNodeNames.toArray(Strings.EMPTY_ARRAY), timeValueSeconds(30))).get(); + new AddVotingConfigExclusionsRequest(excludedNodeNames.toArray(Strings.EMPTY_ARRAY))).get(); } catch (InterruptedException | ExecutionException e) { throw new AssertionError("unexpected", e); } From 5c7a226f29540e46f5d6e26db8722e1f8123db69 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 8 Apr 2020 18:52:56 -0700 Subject: [PATCH 20/29] Migrate some tests that use nodeDescriptions to using nodeNames --- .../AddVotingConfigExclusionsRequest.java | 10 ++-- ...AddVotingConfigExclusionsRequestTests.java | 31 +++++------ ...tAddVotingConfigExclusionsActionTests.java | 52 ++++++++----------- .../cluster/MinimumMasterNodesIT.java | 4 +- .../cluster/SpecificMasterNodesIT.java | 2 +- .../coordination/VotingConfigurationIT.java | 2 +- .../gateway/RecoveryFromGatewayIT.java | 2 +- 7 files changed, 45 insertions(+), 58 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 606ace8544f60..b42422d2497ce 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -54,18 +54,20 @@ public class AddVotingConfigExclusionsRequest extends MasterNodeRequest makeRequest("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), + () -> makeRequestWithNodeDescriptions("not-a-node").resolveVotingConfigExclusions(clusterState)).getMessage(), equalTo("add voting config exclusions request for [not-a-node] matched no master-eligible nodes")); assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } @@ -230,14 +228,10 @@ public void testResolveByNodeNames() { final ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")) .nodes(new Builder().add(node1).add(node2).add(node3).localNodeId(node1.getId())).build(); - assertThat(new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{"nodeName1", "nodeName2"}, TimeValue.ZERO) - .resolveVotingConfigExclusions(clusterState), + assertThat(new AddVotingConfigExclusionsRequest("nodeName1", "nodeName2").resolveVotingConfigExclusions(clusterState), containsInAnyOrder(node1Exclusion, node2Exclusion)); - assertThat(new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{"nodeName1", "unresolvableNodeName"}, TimeValue.ZERO) - .resolveVotingConfigExclusions(clusterState), + assertThat(new AddVotingConfigExclusionsRequest("nodeName1", "unresolvableNodeName").resolveVotingConfigExclusions(clusterState), containsInAnyOrder(node1Exclusion, unresolvableVotingConfigExclusion)); } @@ -316,16 +310,17 @@ public void testResolveAndCheckMaximum() { .coordinationMetaData(CoordinationMetaData.builder().addVotingConfigExclusion(otherNode1Exclusion).build())); final ClusterState clusterState = builder.build(); - assertThat(makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), + assertThat(makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), contains(localNodeExclusion)); assertThat(expectThrows(IllegalArgumentException.class, - () -> makeRequest("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")).getMessage(), + () -> makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")).getMessage(), equalTo("add voting config exclusions request for [_local] would add [1] exclusions to the existing [1] which would " + "exceed the maximum of [1] set by [setting.name]")); assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } - private static AddVotingConfigExclusionsRequest makeRequest(String... descriptions) { - return new AddVotingConfigExclusionsRequest(descriptions); + private static AddVotingConfigExclusionsRequest makeRequestWithNodeDescriptions(String... nodeDescriptions) { + return new AddVotingConfigExclusionsRequest(nodeDescriptions, Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index de5a35dea0fb2..25d669cb79bda 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -152,8 +152,7 @@ public void testWithdrawsVoteFromANode() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, new AddVotingConfigExclusionsRequest("other1"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -162,7 +161,6 @@ public void testWithdrawsVoteFromANode() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { @@ -170,7 +168,7 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1", "other2"}), + new AddVotingConfigExclusionsRequest("other1", "other2"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -180,15 +178,13 @@ public void testWithdrawsVotesFromMultipleNodes() throws InterruptedException { assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), containsInAnyOrder(otherNode1Exclusion, otherNode2Exclusion)); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testWithdrawsVotesFromNodesMatchingWildcard() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other*"}), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, makeRequestWithNodeDescriptions("other*"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -205,8 +201,7 @@ public void testWithdrawsVotesFromAllMasterEligibleNodes() throws InterruptedExc final CountDownLatch countDownLatch = new CountDownLatch(1); clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_all"}), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, makeRequestWithNodeDescriptions("_all"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -223,8 +218,7 @@ public void testWithdrawsVoteFromLocalNode() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_local"}), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, makeRequestWithNodeDescriptions("_local"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -236,6 +230,7 @@ public void testWithdrawsVoteFromLocalNode() throws InterruptedException { contains(localNodeExclusion)); assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } + public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedException { final ClusterState state = clusterService.state(); setState(clusterService, builder(state) @@ -248,8 +243,7 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc final CountDownLatch countDownLatch = new CountDownLatch(1); // no observer to reconfigure - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, TimeValue.ZERO), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, new AddVotingConfigExclusionsRequest("other1"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -259,15 +253,13 @@ public void testReturnsImmediatelyIfVoteAlreadyWithdrawn() throws InterruptedExc assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } - public void testReturnsErrorIfNoMatchingNodes() throws InterruptedException { + public void testReturnsErrorIfNoMatchingNodeDescriptions() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); final SetOnce exceptionHolder = new SetOnce<>(); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"not-a-node"}), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, makeRequestWithNodeDescriptions("not-a-node"), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); @@ -287,7 +279,7 @@ public void testOnlyMatchesMasterEligibleNodes() throws InterruptedException { final SetOnce exceptionHolder = new SetOnce<>(); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"_all", "master:false"}), + makeRequestWithNodeDescriptions("_all", "master:false"), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); @@ -341,9 +333,7 @@ public void testExcludeAbsentNodesByNodeNames() throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{"absent_node"}, TimeValue.timeValueSeconds(30)), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, new AddVotingConfigExclusionsRequest("absent_node"), expectSuccess(e -> { countDownLatch.countDown(); }) @@ -359,8 +349,7 @@ public void testExcludeExistingNodesByNodeNames() throws InterruptedException { clusterStateObserver.waitForNextChange(new AdjustConfigurationForExclusions()); transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{"other1", "other2"}, TimeValue.timeValueSeconds(30)), + new AddVotingConfigExclusionsRequest("other1", "other2"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -384,8 +373,7 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce final CountDownLatch countDownLatch = new CountDownLatch(1); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other1"}), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, new AddVotingConfigExclusionsRequest("other1"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -395,7 +383,6 @@ public void testSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedExce assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); assertThat(clusterService.getClusterApplierService().state().getVotingConfigExclusions(), contains(otherNode1Exclusion)); - assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); } public void testExcludeByNodeIdSucceedsEvenIfAllExclusionsAlreadyAdded() throws InterruptedException { @@ -436,9 +423,7 @@ public void testExcludeByNodeNameSucceedsEvenIfAllExclusionsAlreadyAdded() throw final CountDownLatch countDownLatch = new CountDownLatch(1); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - new String[]{"other1"}, TimeValue.timeValueSeconds(30)), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, new AddVotingConfigExclusionsRequest("other1"), expectSuccess(r -> { assertNotNull(r); countDownLatch.countDown(); @@ -488,8 +473,7 @@ public void testReturnsErrorIfMaximumExclusionCountExceeded() throws Interrupted final CountDownLatch countDownLatch = new CountDownLatch(1); final SetOnce exceptionHolder = new SetOnce<>(); - transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, - new AddVotingConfigExclusionsRequest(new String[]{"other*"}), + transportService.sendRequest(localNode, AddVotingConfigExclusionsAction.NAME, makeRequestWithNodeDescriptions("other*"), expectError(e -> { exceptionHolder.set(e); countDownLatch.countDown(); @@ -601,4 +585,10 @@ public void onTimeout(TimeValue timeout) { throw new AssertionError("unexpected timeout"); } } + + private AddVotingConfigExclusionsRequest makeRequestWithNodeDescriptions(String... nodeDescriptions) { + return new AddVotingConfigExclusionsRequest(nodeDescriptions, Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, TimeValue.timeValueSeconds(30)); + } + } diff --git a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index e9043528f9c0c..28fa48a8ed2d7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -123,7 +123,7 @@ public void testTwoNodesNoMasterBlock() throws Exception { String masterNode = internalCluster().getMasterName(); String otherNode = node1Name.equals(masterNode) ? node2Name : node1Name; logger.info("--> add voting config exclusion for non-master node, to be sure it's not elected"); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{otherNode})).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(otherNode)).get(); logger.info("--> stop master node, no master block should appear"); Settings masterDataPathSettings = internalCluster().dataPathSettings(masterNode); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNode)); @@ -170,7 +170,7 @@ public void testTwoNodesNoMasterBlock() throws Exception { masterNode = internalCluster().getMasterName(); otherNode = node1Name.equals(masterNode) ? node2Name : node1Name; logger.info("--> add voting config exclusion for master node, to be sure it's not elected"); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{masterNode})).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(masterNode)).get(); logger.info("--> stop non-master node, no master block should appear"); Settings otherNodeDataPathSettings = internalCluster().dataPathSettings(otherNode); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(otherNode)); diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index 0e8f0f3f7e193..b8a89188bf74e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -115,7 +115,7 @@ public void testElectOnlyBetweenMasterNodes() throws Exception { logger.info("--> closing master node (1)"); client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(new String[]{masterNodeName})).get(); + new AddVotingConfigExclusionsRequest(masterNodeName)).get(); // removing the master from the voting configuration immediately triggers the master to step down assertBusy(() -> { assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState() diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java index c3c3abf8866b7..e8f1fe8aabaeb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java @@ -55,7 +55,7 @@ public void testAbdicateAfterVotingConfigExclusionAdded() throws ExecutionExcept logger.info("--> excluding master node {}", originalMaster); client().execute(AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(new String[]{originalMaster})).get(); + new AddVotingConfigExclusionsRequest(originalMaster)).get(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get(); assertNotEquals(originalMaster, internalCluster().getMasterName()); } diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index f20bee58d0487..4bc7e80f3c995 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -309,7 +309,7 @@ public void testTwoNodeFirstNodeCleared() throws Exception { Map primaryTerms = assertAndCapturePrimaryTerms(null); - client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{firstNode})).get(); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(firstNode)).get(); internalCluster().fullRestart(new RestartCallback() { @Override From c6fbce8534d44b6b073415412bccc7aa14c77831 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 8 Apr 2020 19:27:02 -0700 Subject: [PATCH 21/29] Fix style --- .../AddVotingConfigExclusionsRequest.java | 25 ++++++++----------- .../coordination/JoinTaskExecutor.java | 2 +- ...AddVotingConfigExclusionsRequestTests.java | 3 ++- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index b42422d2497ce..c340779bc4df8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -66,8 +66,10 @@ public AddVotingConfigExclusionsRequest(String... nodeNames) { * Construct a request to add voting config exclusions for master-eligible nodes matching the given descriptions, and wait for these * nodes to be removed from the voting configuration. * @param nodeDescriptions Descriptions of the nodes whose exclusions to add - see {@link DiscoveryNodes#resolveNodes(String...)}. - * @param nodeIds Ids of the nodes whose exclusions to add - see {@link AddVotingConfigExclusionsRequest#resolveVotingConfigExclusions(ClusterState) - * @param nodeNames Names of the nodes whose exclusions to add - see {@link AddVotingConfigExclusionsRequest#resolveVotingConfigExclusions(ClusterState) + * @param nodeIds Ids of the nodes whose exclusions to add - see + * {@link AddVotingConfigExclusionsRequest#resolveVotingConfigExclusions(ClusterState)}. + * @param nodeNames Names of the nodes whose exclusions to add - see + * {@link AddVotingConfigExclusionsRequest#resolveVotingConfigExclusions(ClusterState)}. * @param timeout How long to wait for the added exclusions to take effect and be removed from the voting configuration. */ public AddVotingConfigExclusionsRequest(String[] nodeDescriptions, String[] nodeIds, String[] nodeNames, TimeValue timeout) { @@ -75,7 +77,7 @@ public AddVotingConfigExclusionsRequest(String[] nodeDescriptions, String[] node throw new IllegalArgumentException("timeout [" + timeout + "] must be non-negative"); } - if(noneOrMoreThanOneIsSet(nodeDescriptions, nodeIds, nodeNames)) { + if (noneOrMoreThanOneIsSet(nodeDescriptions, nodeIds, nodeNames)) { throw new IllegalArgumentException("Please set node identifiers correctly. " + "One and only one of [node_name], [node_names] and [node_ids] has to be set"); } @@ -96,8 +98,7 @@ public AddVotingConfigExclusionsRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_8_0_0)) { nodeIds = in.readStringArray(); nodeNames = in.readStringArray(); - } - else { + } else { nodeIds = Strings.EMPTY_ARRAY; nodeNames = Strings.EMPTY_ARRAY; } @@ -129,8 +130,7 @@ Set resolveVotingConfigExclusions(ClusterState currentSta if (discoveryNode.isMasterNode()) { newVotingConfigExclusions.add(new VotingConfigExclusion(discoveryNode)); } - } - else { + } else { newVotingConfigExclusions.add(new VotingConfigExclusion(nodeId, VotingConfigExclusion.MISSING_VALUE_MARKER)); } } @@ -144,8 +144,7 @@ Set resolveVotingConfigExclusions(ClusterState currentSta if (discoveryNode.isMasterNode()) { newVotingConfigExclusions.add(new VotingConfigExclusion(discoveryNode)); } - } - else { + } else { newVotingConfigExclusions.add(new VotingConfigExclusion(VotingConfigExclusion.MISSING_VALUE_MARKER, nodeName)); } } @@ -171,13 +170,11 @@ Set resolveVotingConfigExclusionsAndCheckMaximum(ClusterS } private boolean noneOrMoreThanOneIsSet(String[] deprecatedNodeDescription, String[] nodeIds, String[] nodeNames) { - if(deprecatedNodeDescription.length > 0) { + if (deprecatedNodeDescription.length > 0) { return nodeIds.length > 0 || nodeNames.length > 0; - } - else if (nodeIds.length > 0) { + } else if (nodeIds.length > 0) { return nodeNames.length > 0; - } - else { + } else { return nodeNames.length > 0 == false; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java index afd29adf4ab46..aa977a546dbc8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java @@ -149,7 +149,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo nodesChanged = true; minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion()); maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion()); - if(node.isMasterNode()) { + if (node.isMasterNode()) { joiniedNodeNameIds.put(node.getName(), node.getId()); } } catch (IllegalArgumentException | IllegalStateException e) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index 4f6880068a1c5..0d14f4b2c616a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -313,7 +313,8 @@ public void testResolveAndCheckMaximum() { assertThat(makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 2, "setting.name"), contains(localNodeExclusion)); assertThat(expectThrows(IllegalArgumentException.class, - () -> makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")).getMessage(), + () -> makeRequestWithNodeDescriptions("_local").resolveVotingConfigExclusionsAndCheckMaximum(clusterState, 1, "setting.name")) + .getMessage(), equalTo("add voting config exclusions request for [_local] would add [1] exclusions to the existing [1] which would " + "exceed the maximum of [1] set by [setting.name]")); assertWarnings(AddVotingConfigExclusionsRequest.DEPRECATION_MESSAGE); From fcd39120b5cd35a42ba6c53a3de6a349cb119fff Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 8 Apr 2020 19:46:52 -0700 Subject: [PATCH 22/29] Merge branch 'master' into issue-47990 --- .ci/bwcVersions | 2 + .ci/java-versions.properties | 2 +- .ci/jobs.t/defaults.yml | 1 + .ci/os.ps1 | 4 +- .ci/os.sh | 18 +- .ci/packer_cache.sh | 1 + .../runConfigurations/Debug_Elasticsearch.xml | 6 +- .../allocation/AllocationBenchmark.java | 14 +- .../routing/allocation/Allocators.java | 10 +- build.gradle | 47 +- buildSrc/build.gradle | 2 + .../formatterConfig.xml | 4 +- .../elasticsearch/gradle/BuildPlugin.groovy | 1 + .../gradle/plugin/PluginBuildPlugin.groovy | 5 +- .../gradle/precommit/PrecommitTasks.groovy | 22 +- .../gradle/test/RestIntegTestTask.groovy | 97 - .../test/StandaloneRestTestPlugin.groovy | 13 +- ...emPropertyCommandLineArgumentProvider.java | 12 +- .../gradle/checkstyle/SnippetLengthCheck.java | 103 + .../precommit/DependencyLicensesTask.java | 3 +- .../gradle/precommit/ThirdPartyAuditTask.java | 2 +- .../gradle/test/DistroTestPlugin.java | 16 + .../elasticsearch/gradle/test/Fixture.java} | 7 +- .../gradle/test/RestIntegTestTask.java | 104 + .../org/elasticsearch/gradle/util/Util.java | 10 + buildSrc/src/main/resources/checkstyle.xml | 6 +- .../src/main/resources/minimumCompilerVersion | 2 +- .../src/main/resources/minimumGradleVersion | 2 +- .../checkstyle/SnipptLengthCheckTests.java | 87 + buildSrc/version.properties | 3 + .../benchmark/rest/RestClientBenchmark.java | 6 +- .../client/AsyncSearchRequestConverters.java | 18 +- .../elasticsearch/client/ClusterClient.java | 115 ++ .../client/ClusterRequestConverters.java | 59 + .../client/GetAliasesResponse.java | 24 +- .../elasticsearch/client/IndicesClient.java | 127 +- .../client/IndicesRequestConverters.java | 61 +- .../client/MLRequestConverters.java | 12 + .../client/MachineLearningClient.java | 44 + .../client/RestHighLevelClient.java | 13 +- .../client/TasksRequestConverters.java | 3 + .../StringStatsAggregationBuilder.java | 22 +- .../TopMetricsAggregationBuilder.java | 7 +- .../asyncsearch/SubmitAsyncSearchRequest.java | 60 +- .../client/core/IndexerJobStats.java | 1 - .../ComponentTemplatesExistRequest.java | 40 + .../DeleteComponentTemplateRequest.java | 35 + .../indices/DeleteIndexTemplateV2Request.java | 35 + .../indices/GetComponentTemplatesRequest.java | 80 + .../GetComponentTemplatesResponse.java | 108 + .../indices/GetFieldMappingsResponse.java | 38 +- .../client/indices/GetIndexResponse.java | 42 +- .../indices/GetIndexTemplateV2Request.java | 80 + .../indices/GetIndexTemplatesResponse.java | 32 +- .../indices/GetIndexTemplatesV2Response.java | 108 + .../client/indices/GetMappingsResponse.java | 12 +- ...taData.java => IndexTemplateMetadata.java} | 80 +- .../indices/IndexTemplateV2ExistRequest.java | 40 + .../indices/PutComponentTemplateRequest.java | 100 + .../indices/PutIndexTemplateV2Request.java | 100 + .../client/ml/EstimateModelMemoryRequest.java | 110 + .../ml/EstimateModelMemoryResponse.java | 80 + .../stats/classification/Hyperparameters.java | 142 +- .../dataframe/stats/common/MemoryUsage.java | 15 +- .../stats/regression/Hyperparameters.java | 146 +- .../MlInferenceNamedXContentProvider.java | 11 + .../inference/NamedXContentObjectHelper.java | 10 + .../ml/inference/TrainedModelConfig.java | 30 +- .../trainedmodel/ClassificationConfig.java | 129 ++ .../trainedmodel/InferenceConfig.java | 10 +- .../trainedmodel/RegressionConfig.java | 104 + .../ml/job/results/CategoryDefinition.java | 43 +- .../client/rollup/GetRollupCapsRequest.java | 4 +- .../client/tasks/CancelTasksRequest.java | 24 +- .../transforms/TransformIndexerStats.java | 207 +- ...cherMetaData.java => WatcherMetadata.java} | 6 +- .../client/watcher/WatcherStatsResponse.java | 16 +- .../AsyncSearchRequestConvertersTests.java | 18 +- .../elasticsearch/client/ClusterClientIT.java | 55 + .../client/GetAliasesResponseTests.java | 22 +- .../HighLevelRestClientCompressionIT.java | 54 + .../elasticsearch/client/IndicesClientIT.java | 192 +- .../client/MLRequestConvertersTests.java | 21 + .../client/MachineLearningIT.java | 31 +- .../client/RequestConvertersTests.java | 3 +- .../client/RestHighLevelClientTests.java | 71 +- .../org/elasticsearch/client/SearchIT.java | 30 +- .../org/elasticsearch/client/SnapshotIT.java | 25 +- .../client/TasksRequestConvertersTests.java | 14 +- .../org/elasticsearch/client/WatcherIT.java | 4 +- .../client/asyncsearch/AsyncSearchIT.java | 2 +- .../client/ccr/CcrStatsResponseTests.java | 2 +- .../client/core/CountResponseTests.java | 4 +- .../AsyncSearchDocumentationIT.java | 238 +++ .../documentation/ILMDocumentationIT.java | 10 +- .../IndicesClientDocumentationIT.java | 42 +- .../MlClientDocumentationIT.java | 65 + .../SnapshotClientDocumentationIT.java | 16 +- .../TasksClientDocumentationIT.java | 3 +- .../client/enrich/StatsResponseTests.java | 2 +- .../client/eql/EqlSearchResponseTests.java | 2 +- .../IndexLifecycleExplainResponseTests.java | 2 +- .../GetComponentTemplatesResponseTests.java | 135 ++ .../GetFieldMappingsResponseTests.java | 14 +- .../client/indices/GetIndexResponseTests.java | 22 +- .../GetIndexTemplatesResponseTests.java | 52 +- .../GetIndexTemplatesV2ResponseTests.java | 89 + .../indices/GetMappingsResponseTests.java | 10 +- .../classification/HyperparametersTests.java | 6 +- .../stats/common/MemoryUsageTests.java | 9 +- .../regression/HyperparametersTests.java | 6 +- .../ml/inference/TrainedModelConfigTests.java | 11 +- .../ClassificationConfigTests.java | 51 + .../trainedmodel/RegressionConfigTests.java | 50 + .../job/results/CategoryDefinitionTests.java | 11 + .../rollup/GetRollupCapsRequestTests.java | 6 +- .../TransformIndexerStatsTests.java | 49 +- .../transforms/hlrc/TransformStatsTests.java | 8 +- .../hlrc/DateHistogramGroupSourceTests.java | 9 +- .../watcher/WatcherStatsResponseTests.java | 10 +- distribution/bwc/build.gradle | 4 + .../src/docker/config/log4j2.properties | 10 +- .../src/docker/config/oss/log4j2.properties | 8 +- .../resources/rest-api-spec/test/11_nodes.yml | 4 +- distribution/packages/build.gradle | 17 +- distribution/src/config/log4j2.properties | 63 +- distribution/tools/keystore-cli/build.gradle | 2 +- .../settings/AddFileKeyStoreCommand.java | 54 +- .../settings/AddStringKeyStoreCommand.java | 88 +- .../RemoveSettingKeyStoreCommand.java | 6 +- .../settings/AddFileKeyStoreCommandTests.java | 36 +- .../AddStringKeyStoreCommandTests.java | 36 +- .../tools/launchers/JvmOptionsParser.java | 4 +- .../tools/launchers/SystemJvmOptions.java | 12 + distribution/tools/plugin-cli/build.gradle | 4 +- .../plugins/InstallPluginCommand.java | 7 +- docs/build.gradle | 36 + .../high-level/asyncsearch/delete.asciidoc | 68 + .../high-level/asyncsearch/get.asciidoc | 87 + .../high-level/asyncsearch/submit.asciidoc | 94 + .../ml/estimate-model-memory.asciidoc | 42 + .../high-level/ml/put-trained-model.asciidoc | 2 + .../high-level/supported-apis.asciidoc | 18 + .../high-level/tasks/cancel_tasks.asciidoc | 4 +- ...painless-ingest-processor-context.asciidoc | 101 +- .../painless-reindex-context.asciidoc | 6 - .../painless-update-by-query-context.asciidoc | 3 - .../painless-update-context.asciidoc | 3 - docs/reference/aggregations/metrics.asciidoc | 2 +- .../metrics/t-test-aggregation.asciidoc | 114 + .../metrics/top-metrics-aggregation.asciidoc | 4 +- docs/reference/analysis/normalizers.asciidoc | 6 +- .../keyword-marker-tokenfilter.asciidoc | 419 +++- .../keyword-repeat-tokenfilter.asciidoc | 445 +++- .../tokenizers/keyword-tokenizer.asciidoc | 48 + .../apis/autoscaling-apis.asciidoc | 6 + .../apis/delete-autoscaling-policy.asciidoc | 64 + .../apis/get-autoscaling-policy.asciidoc | 67 + .../apis/put-autoscaling-policy.asciidoc | 67 + docs/reference/cat/tasks.asciidoc | 10 - docs/reference/cat/transforms.asciidoc | 177 +- .../ccr/apis/follow/get-follow-info.asciidoc | 141 +- .../ccr/apis/follow/get-follow-stats.asciidoc | 213 +- .../reference/ccr/apis/get-ccr-stats.asciidoc | 56 +- docs/reference/cluster/nodes-stats.asciidoc | 1727 +++++++++++----- docs/reference/cluster/stats.asciidoc | 530 +++-- docs/reference/cluster/tasks.asciidoc | 5 + docs/reference/commands/keystore.asciidoc | 68 +- docs/reference/docs/multi-get.asciidoc | 6 +- docs/reference/eql/functions.asciidoc | 392 ++++ docs/reference/eql/index.asciidoc | 2 + docs/reference/eql/limitations.asciidoc | 5 +- docs/reference/eql/syntax.asciidoc | 58 + docs/reference/how-to/search-speed.asciidoc | 160 +- .../reference/ilm/apis/get-lifecycle.asciidoc | 4 +- .../ilm/getting-started-ilm.asciidoc | 114 +- .../reference/ilm/policy-definitions.asciidoc | 48 +- .../ilm/update-lifecycle-policy.asciidoc | 4 +- docs/reference/ingest/ingest-node.asciidoc | 1 - .../ingest/processors/inference.asciidoc | 105 +- .../licensing/update-license.asciidoc | 2 +- .../mapping/params/normalizer.asciidoc | 6 +- .../mapping/types/date_nanos.asciidoc | 2 +- docs/reference/mapping/types/point.asciidoc | 99 + .../migration/migrate_8_0/settings.asciidoc | 14 + .../apis/estimate-model-memory.asciidoc | 94 + .../apis/find-file-structure.asciidoc | 2 +- .../apis/get-bucket.asciidoc | 36 +- .../apis/get-category.asciidoc | 10 + .../apis/get-datafeed-stats.asciidoc | 38 +- .../apis/get-job-stats.asciidoc | 158 +- .../apis/get-snapshot.asciidoc | 45 +- .../ml/anomaly-detection/apis/ml-api.asciidoc | 2 + .../apis/post-calendar-event.asciidoc | 19 +- .../apis/preview-datafeed.asciidoc | 6 + .../apis/put-datafeed.asciidoc | 4 + .../anomaly-detection/apis/put-job.asciidoc | 155 +- .../apis/update-datafeed.asciidoc | 7 + .../apis/update-job.asciidoc | 106 +- .../apis/delete-dfanalytics.asciidoc | 2 +- .../delete-inference-trained-model.asciidoc | 2 +- .../apis/explain-dfanalytics.asciidoc | 22 +- .../apis/get-dfanalytics-stats.asciidoc | 3 +- .../apis/get-dfanalytics.asciidoc | 2 +- .../apis/get-inference-trained-model.asciidoc | 2 +- .../apis/put-dfanalytics.asciidoc | 173 +- .../df-analytics/apis/put-inference.asciidoc | 382 ++-- .../apis/start-dfanalytics.asciidoc | 2 +- .../apis/stop-dfanalytics.asciidoc | 2 +- docs/reference/ml/ml-shared.asciidoc | 681 ++++-- .../modules/indices/recovery.asciidoc | 37 +- docs/reference/modules/node.asciidoc | 88 +- .../modules/remote-clusters.asciidoc | 8 +- docs/reference/modules/threadpool.asciidoc | 27 +- docs/reference/query-dsl/range-query.asciidoc | 16 +- .../query-dsl/shape-queries.asciidoc | 15 +- docs/reference/redirects.asciidoc | 10 +- docs/reference/rest-api/common-parms.asciidoc | 103 +- docs/reference/rest-api/index.asciidoc | 7 + docs/reference/rollup/apis/get-job.asciidoc | 38 +- docs/reference/rollup/apis/put-job.asciidoc | 214 +- docs/reference/search/async-search.asciidoc | 17 +- .../reference/search/suggesters/misc.asciidoc | 2 + .../apis/clear-cache.asciidoc | 76 + .../apis/get-stats.asciidoc | 76 + .../apis/mount-snapshot.asciidoc | 126 ++ .../apis/searchable-snapshots-apis.asciidoc | 16 + docs/reference/settings/ml-settings.asciidoc | 23 +- .../settings/security-settings.asciidoc | 4 +- .../settings/transform-settings.asciidoc | 21 +- docs/reference/slm/apis/slm-put.asciidoc | 117 +- docs/reference/slm/slm-retention.asciidoc | 2 +- .../sql/endpoints/odbc/configuration.asciidoc | 57 +- .../sql/functions/date-time.asciidoc | 42 + docs/reference/sql/functions/index.asciidoc | 1 + .../transform/apis/delete-transform.asciidoc | 13 +- .../apis/get-transform-stats.asciidoc | 272 ++- .../transform/apis/get-transform.asciidoc | 12 +- .../transform/apis/preview-transform.asciidoc | 108 +- .../transform/apis/put-transform.asciidoc | 80 +- .../transform/apis/start-transform.asciidoc | 12 +- .../transform/apis/stop-transform.asciidoc | 12 +- .../transform/apis/update-transform.asciidoc | 68 +- docs/reference/transform/examples.asciidoc | 85 +- .../transform/painless-examples.asciidoc | 260 ++- .../upgrade/rolling_upgrade.asciidoc | 4 +- gradle/forbidden-dependencies.gradle | 27 + gradle/formatting.gradle | 193 ++ gradle/wrapper/gradle-wrapper.jar | Bin 58695 -> 58694 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- .../elasticsearch/common/unit/TimeValue.java | 31 +- .../common/unit/TimeValueTests.java | 22 + .../matrix/stats/InternalMatrixStats.java | 11 +- .../stats/MatrixStatsAggregationBuilder.java | 22 +- .../matrix/stats/MatrixStatsAggregator.java | 11 +- .../stats/MatrixStatsAggregatorFactory.java | 33 +- .../ArrayValuesSourceAggregationBuilder.java | 208 +- .../ArrayValuesSourceAggregatorFactory.java | 34 +- .../support/ArrayValuesSourceParser.java | 19 +- .../stats/InternalMatrixStatsTests.java | 22 +- .../common/CommonAnalysisPluginTests.java | 14 +- .../common/CompoundAnalysisTests.java | 6 +- .../common/EdgeNGramTokenizerTests.java | 4 +- .../common/MultiplexerTokenFilterTests.java | 6 +- .../common/NGramTokenizerFactoryTests.java | 4 +- .../PatternCaptureTokenFilterTests.java | 4 +- .../PredicateTokenScriptFilterTests.java | 4 +- .../ScriptedConditionTokenFilterTests.java | 4 +- .../StemmerTokenFilterFactoryTests.java | 2 +- .../common/SynonymsAnalysisTests.java | 24 +- .../WhitespaceTokenizerFactoryTests.java | 6 +- ...DelimiterGraphTokenFilterFactoryTests.java | 6 +- modules/geo/build.gradle | 23 + .../java/org/elasticsearch/geo/GeoPlugin.java | 37 + .../geo/GeoClientYamlTestSuiteIT.java | 38 + .../java/org/elasticsearch/geo/GeoTests.java | 28 + .../rest-api-spec/test/geo_shape/10_basic.yml | 59 + .../ingest/common/DateIndexNameProcessor.java | 2 +- .../ingest/common/ScriptProcessor.java | 8 +- .../ingest/common/AppendProcessorTests.java | 12 +- .../ingest/common/ForEachProcessorTests.java | 2 +- .../ingest/common/ScriptProcessorTests.java | 24 - .../ingest/common/SetProcessorTests.java | 16 +- modules/kibana/build.gradle | 31 + .../elasticsearch/kibana/KibanaPlugin.java | 143 ++ .../kibana/KibanaPluginTests.java | 49 + .../kibana/KibanaSystemIndexIT.java | 260 +++ .../expression/DateObjectValueSource.java | 4 +- .../expression/ExpressionScriptEngine.java | 3 +- .../expression/GeoEmptyValueSource.java | 4 +- .../expression/GeoLatitudeValueSource.java | 4 +- .../expression/GeoLongitudeValueSource.java | 4 +- .../mustache/SearchTemplateResponseTests.java | 2 +- .../org/elasticsearch/painless/Compiler.java | 11 +- .../elasticsearch/painless/antlr/Walker.java | 16 +- .../elasticsearch/painless/ir/ClassNode.java | 32 +- .../painless/node/AExpression.java | 7 - .../elasticsearch/painless/node/ANode.java | 88 +- .../painless/node/DResolvedType.java | 5 - .../painless/node/DUnresolvedType.java | 5 - .../painless/node/EAssignment.java | 23 - .../elasticsearch/painless/node/EBinary.java | 10 +- .../elasticsearch/painless/node/EBool.java | 10 +- .../elasticsearch/painless/node/EBoolean.java | 11 +- .../painless/node/ECallLocal.java | 9 +- .../painless/node/ECapturingFunctionRef.java | 5 - .../elasticsearch/painless/node/EComp.java | 10 +- .../painless/node/EConditional.java | 9 +- .../painless/node/EConstant.java | 13 +- .../elasticsearch/painless/node/EDecimal.java | 14 +- .../elasticsearch/painless/node/EElvis.java | 9 +- .../painless/node/EExplicit.java | 10 +- .../painless/node/EFunctionRef.java | 5 - .../painless/node/EInstanceof.java | 10 +- .../elasticsearch/painless/node/ELambda.java | 5 - .../painless/node/EListInit.java | 10 +- .../elasticsearch/painless/node/EMapInit.java | 10 +- .../painless/node/ENewArray.java | 11 +- .../painless/node/ENewArrayFunctionRef.java | 66 +- .../elasticsearch/painless/node/ENewObj.java | 7 - .../elasticsearch/painless/node/ENull.java | 11 +- .../elasticsearch/painless/node/ENumeric.java | 17 +- .../elasticsearch/painless/node/ERegex.java | 44 +- .../elasticsearch/painless/node/EStatic.java | 10 +- .../elasticsearch/painless/node/EString.java | 10 +- .../elasticsearch/painless/node/EUnary.java | 10 +- .../painless/node/EVariable.java | 9 +- .../elasticsearch/painless/node/PBrace.java | 9 +- .../painless/node/PCallInvoke.java | 7 - .../elasticsearch/painless/node/PField.java | 12 +- .../painless/node/PSubArrayLength.java | 5 - .../painless/node/PSubBrace.java | 5 - .../painless/node/PSubCallInvoke.java | 6 - .../painless/node/PSubDefArray.java | 5 - .../painless/node/PSubDefCall.java | 5 - .../painless/node/PSubDefField.java | 5 - .../painless/node/PSubField.java | 5 - .../painless/node/PSubListShortcut.java | 5 - .../painless/node/PSubMapShortcut.java | 5 - .../painless/node/PSubNullSafeCallInvoke.java | 5 - .../painless/node/PSubNullSafeField.java | 5 - .../painless/node/PSubShortcut.java | 5 - .../elasticsearch/painless/node/SBlock.java | 7 - .../elasticsearch/painless/node/SBreak.java | 5 - .../elasticsearch/painless/node/SCatch.java | 5 - .../elasticsearch/painless/node/SClass.java | 47 +- .../painless/node/SContinue.java | 5 - .../painless/node/SDeclBlock.java | 7 - .../painless/node/SDeclaration.java | 8 - .../org/elasticsearch/painless/node/SDo.java | 5 - .../elasticsearch/painless/node/SEach.java | 5 - .../painless/node/SExpression.java | 9 - .../org/elasticsearch/painless/node/SFor.java | 17 - .../painless/node/SFunction.java | 71 +- .../org/elasticsearch/painless/node/SIf.java | 5 - .../elasticsearch/painless/node/SIfElse.java | 8 - .../elasticsearch/painless/node/SReturn.java | 5 - .../painless/node/SSubEachArray.java | 6 - .../painless/node/SSubEachIterable.java | 6 - .../elasticsearch/painless/node/SThrow.java | 5 - .../org/elasticsearch/painless/node/STry.java | 7 - .../elasticsearch/painless/node/SWhile.java | 5 - .../painless/symbol/ScriptRoot.java | 21 +- .../painless/ContextExampleTests.java | 42 +- .../elasticsearch/painless/FactoryTests.java | 3 +- .../elasticsearch/painless/FunctionTests.java | 2 +- .../painless/ScriptTestCase.java | 2 +- .../painless/WhenThingsGoWrongTests.java | 43 + .../painless/node/NodeToStringTests.java | 889 -------- .../index/mapper/ScaledFloatFieldMapper.java | 2 + .../mapper/ScaledFloatFieldTypeTests.java | 4 +- .../query/RankFeatureQueryBuilderTests.java | 7 +- .../ChildrenAggregationBuilder.java | 54 +- .../ChildrenAggregatorFactory.java | 31 +- .../ChildrenToParentAggregator.java | 11 +- .../join/aggregations/InternalChildren.java | 9 +- .../join/aggregations/InternalParent.java | 9 +- .../ParentAggregationBuilder.java | 53 +- .../aggregations/ParentAggregatorFactory.java | 30 +- .../aggregations/ParentJoinAggregator.java | 9 +- .../ParentToChildrenAggregator.java | 11 +- .../join/mapper/ParentJoinFieldMapper.java | 2 +- .../join/aggregations/ChildrenIT.java | 6 +- .../join/aggregations/ChildrenTests.java | 5 +- .../ChildrenToParentAggregatorTests.java | 11 +- .../aggregations/InternalChildrenTests.java | 5 +- .../aggregations/InternalParentTests.java | 13 +- .../join/aggregations/ParentTests.java | 5 +- .../ParentToChildrenAggregatorTests.java | 4 +- .../join/query/HasChildQueryBuilderTests.java | 14 +- .../query/HasParentQueryBuilderTests.java | 14 +- .../join/query/ParentIdQueryBuilderTests.java | 13 +- modules/percolator/build.gradle | 16 +- .../percolator/PercolatorFieldMapper.java | 10 +- .../PercolatorHighlightSubFetchPhase.java | 2 +- .../PercolatorMatchedSlotSubFetchPhase.java | 2 +- .../percolator/CandidateQueryTests.java | 6 +- .../PercolateQueryBuilderTests.java | 5 +- .../PercolatorFieldMapperTests.java | 18 +- .../percolator/PercolatorQuerySearchIT.java | 23 +- .../percolator/QueryBuilderStoreTests.java | 4 +- .../DiscountedCumulativeGainTests.java | 6 +- .../rankeval/ExpectedReciprocalRankTests.java | 2 +- .../rankeval/MeanReciprocalRankTests.java | 2 +- .../index/rankeval/PrecisionAtKTests.java | 6 +- .../index/rankeval/RankEvalResponseTests.java | 2 +- .../index/rankeval/RatedSearchHitTests.java | 6 +- .../index/rankeval/RecallAtKTests.java | 4 +- .../AbstractAsyncBulkByScrollAction.java | 20 +- .../reindex/AsyncBulkByScrollActionTests.java | 23 +- .../ClientScrollableHitSourceTests.java | 2 +- .../reindex/DeleteByQueryBasicTests.java | 4 +- .../ReindexSourceTargetValidationTests.java | 16 +- .../blobstore/url/URLBlobContainer.java | 6 +- .../repositories/url/URLRepository.java | 4 +- .../repositories/url/URLRepositoryTests.java | 22 +- .../tasksplugin/TasksPlugin.java | 3 +- .../tasksplugin/TasksPluginTests.java | 3 +- .../netty4/ByteBufBytesReference.java | 104 - .../netty4/Netty4MessageChannelHandler.java | 28 +- .../netty4/Netty4SizeHeaderFrameDecoder.java | 64 - .../transport/netty4/Netty4Transport.java | 6 +- .../transport/netty4/Netty4Utils.java | 41 +- .../rest/discovery/Zen2RestApiIT.java | 6 +- .../netty4/ByteBufBytesReferenceTests.java | 84 - .../transport/netty4/Netty4UtilsTests.java | 4 +- .../index/analysis/IcuAnalyzerTests.java | 10 +- .../analysis/IcuTokenizerFactoryTests.java | 4 +- .../index/analysis/KuromojiAnalysisTests.java | 6 +- .../index/analysis/NoriAnalysisTests.java | 8 +- .../AnalysisPhoneticFactoryTests.java | 4 +- .../analysis/SimplePhoneticAnalysisTests.java | 4 +- .../analysis/AnalysisPolishFactoryTests.java | 14 +- .../customsuggester/CustomSuggestion.java | 6 +- .../example/CustomAuthorizationEngine.java | 8 +- .../CustomAuthorizationEngineTests.java | 14 +- plugins/ingest-attachment/build.gradle | 14 +- .../licenses/SparseBitSet-1.2.jar.sha1 | 1 + .../licenses/SparseBitSet-LICENSE.txt | 202 ++ .../licenses/SparseBitSet-NOTICE.txt} | 4 +- .../licenses/commons-compress-1.18.jar.sha1 | 1 - .../licenses/commons-compress-1.19.jar.sha1 | 1 + .../licenses/commons-math3-3.6.1.jar.sha1 | 1 + .../licenses/commons-math3-LICENSE.txt | 202 ++ .../licenses/commons-math3-NOTICE.txt | 5 + .../licenses/fontbox-2.0.16.jar.sha1 | 1 - .../licenses/fontbox-2.0.19.jar.sha1 | 1 + .../licenses/pdfbox-2.0.16.jar.sha1 | 1 - .../licenses/pdfbox-2.0.19.jar.sha1 | 1 + .../licenses/poi-4.0.1.jar.sha1 | 1 - .../licenses/poi-4.1.2.jar.sha1 | 1 + .../licenses/poi-ooxml-4.0.1.jar.sha1 | 1 - .../licenses/poi-ooxml-4.1.2.jar.sha1 | 1 + .../licenses/poi-ooxml-schemas-4.0.1.jar.sha1 | 1 - .../licenses/poi-ooxml-schemas-4.1.2.jar.sha1 | 1 + .../licenses/poi-scratchpad-4.0.1.jar.sha1 | 1 - .../licenses/poi-scratchpad-4.1.2.jar.sha1 | 1 + .../licenses/tika-core-1.22.jar.sha1 | 1 - .../licenses/tika-core-1.24.jar.sha1 | 1 + .../licenses/tika-parsers-1.22.jar.sha1 | 1 - .../licenses/tika-parsers-1.24.jar.sha1 | 1 + plugins/repository-azure/build.gradle | 2 +- .../azure/AzureBlobContainer.java | 31 +- .../repositories/azure/AzureBlobStore.java | 13 +- .../repositories/azure/AzureRepository.java | 6 +- .../azure/AzureStorageService.java | 17 +- .../azure/AzureBlobContainerRetriesTests.java | 179 +- .../azure/AzureRepositorySettingsTests.java | 4 +- plugins/repository-gcs/build.gradle | 61 +- .../licenses/httpclient-4.5.10.jar.sha1 | 1 - .../licenses/httpclient-LICENSE.txt | 558 ----- .../licenses/httpcore-4.4.12.jar.sha1 | 1 - .../gcs/GoogleCloudStorageBlobContainer.java | 6 +- .../gcs/GoogleCloudStorageBlobStore.java | 12 +- .../gcs/GoogleCloudStorageRepository.java | 8 +- ...eCloudStorageBlobStoreRepositoryTests.java | 22 +- plugins/repository-hdfs/build.gradle | 4 +- .../hadoop-annotations-2.8.1.jar.sha1 | 1 - .../hadoop-annotations-2.8.5.jar.sha1 | 1 + .../licenses/hadoop-auth-2.8.1.jar.sha1 | 1 - .../licenses/hadoop-auth-2.8.5.jar.sha1 | 1 + .../licenses/hadoop-client-2.8.1.jar.sha1 | 1 - .../licenses/hadoop-client-2.8.5.jar.sha1 | 1 + .../licenses/hadoop-common-2.8.1.jar.sha1 | 1 - .../licenses/hadoop-common-2.8.5.jar.sha1 | 1 + .../licenses/hadoop-hdfs-2.8.1.jar.sha1 | 1 - .../licenses/hadoop-hdfs-2.8.5.jar.sha1 | 1 + .../hadoop-hdfs-client-2.8.1.jar.sha1 | 1 - .../hadoop-hdfs-client-2.8.5.jar.sha1 | 1 + .../repositories/hdfs/HdfsBlobContainer.java | 12 +- .../repositories/hdfs/HdfsRepository.java | 4 +- .../repositories/hdfs/HdfsTests.java | 4 +- plugins/repository-s3/build.gradle | 7 +- .../repositories/s3/S3BlobContainer.java | 36 +- .../repositories/s3/S3BlobStore.java | 12 +- .../repositories/s3/S3ClientSettings.java | 6 +- .../repositories/s3/S3Repository.java | 16 +- .../repositories/s3/S3RepositoryPlugin.java | 4 +- .../s3/S3RetryingInputStream.java | 36 +- .../repositories/s3/S3Service.java | 28 +- .../s3/RepositoryCredentialsTests.java | 4 +- .../s3/S3BlobContainerRetriesTests.java | 172 +- .../s3/S3BlobStoreRepositoryTests.java | 4 +- .../s3/S3ClientSettingsTests.java | 6 +- .../repositories/s3/S3RepositoryTests.java | 24 +- .../s3/S3RepositoryThirdPartyTests.java | 4 +- .../elasticsearch/http/nio/ByteBufUtils.java | 280 +-- .../transport/nio/NioTransport.java | 4 +- .../transport/nio/TcpReadWriteHandler.java | 40 +- .../elasticsearch/DieWithDignityPlugin.java | 15 +- .../qa/die_with_dignity/DieWithDignityIT.java | 35 +- .../metadata/EvilSystemPropertyTests.java | 6 +- .../common/logging/EvilLoggerTests.java | 8 +- .../upgrades/FullClusterRestartIT.java | 28 +- qa/logging-config/build.gradle | 5 +- qa/logging-config/custom-log4j2.properties | 158 +- .../common/logging/ESJsonLayoutTests.java | 4 +- .../common/logging/JsonLoggerTests.java | 172 +- .../custom_logging/CustomLoggingConfigIT.java | 26 +- .../qa/custom_logging/ESJsonLogsConfigIT.java | 67 + .../logging/json_layout/log4j2.properties | 30 +- .../src/test/resources/plugin-security.policy | 1 + .../elasticsearch/backwards/IndexingIT.java | 18 +- qa/multi-cluster-search/build.gradle | 4 +- .../org/elasticsearch/search/CCSDuelIT.java | 19 +- .../test/multi_cluster/10_basic.yml | 129 +- .../test/remote_cluster/10_basic.yml | 12 +- .../packaging/test/DockerTests.java | 9 +- .../elasticsearch/upgrades/RecoveryIT.java | 68 +- .../ingest/IngestDocumentMustacheIT.java | 6 +- .../rest-api-spec/api/cat.aliases.json | 2 +- .../rest-api-spec/api/cat.health.json | 14 +- .../rest-api-spec/api/cat.indices.json | 1 - .../cluster.exists_component_template.json | 35 + .../rest-api-spec/api/cluster.health.json | 1 - .../rest-api-spec/api/delete_by_query.json | 3 +- .../rest-api-spec/api/get_script_context.json | 1 + .../api/get_script_languages.json | 1 + .../api/indices.delete_index_template.json | 35 + .../api/indices.exists_index_template.json | 39 + .../rest-api-spec/api/indices.get_alias.json | 2 +- .../api/indices.get_data_streams.json | 4 +- .../api/indices.get_index_template.json | 45 + .../api/indices.get_settings.json | 5 +- .../api/indices.put_index_template.json | 45 + .../api/snapshot.cleanup_repository.json | 3 +- .../rest-api-spec/api/tasks.cancel.json | 4 + .../rest-api-spec/api/update_by_query.json | 3 +- .../rest-api-spec/test/README.asciidoc | 103 + .../rest-api-spec/test/cat.nodes/10_basic.yml | 4 +- .../test/indices.data_stream/10_basic.yml | 145 +- .../indices.get_index_template/10_basic.yml | 72 + .../20_get_missing.yml | 20 + .../indices.put_index_template/10_basic.yml | 121 ++ .../15_composition.yml | 202 ++ .../test/nodes.info/10_basic.yml | 19 + .../search.aggregation/280_rare_terms.yml | 63 +- .../test/search.aggregation/300_pipeline.yml | 23 + .../test/search.aggregation/30_sig_terms.yml | 2 +- .../330_auto_date_histogram.yml | 73 + .../search.aggregation/340_geo_distance.yml | 84 + .../test/search.aggregation/40_range.yml | 52 + .../test/search/160_exists_query.yml | 75 - rest-api-spec/src/main/resources/schema.json | 245 +++ server/build.gradle | 5 + .../licenses/ecs-logging-core-0.1.3.jar.sha1 | 1 + server/licenses/ecs-logging-core-LICENSE.txt | 1829 +++++++++++++++++ server/licenses/ecs-logging-core-NOTICE.txt | 1081 ++++++++++ .../licenses/log4j2-ecs-layout-0.1.3.jar.sha1 | 1 + server/licenses/log4j2-ecs-layout-LICENSE.txt | 1829 +++++++++++++++++ server/licenses/log4j2-ecs-layout-NOTICE.txt | 1081 ++++++++++ .../elasticsearch/ElasticsearchException.java | 9 +- .../main/java/org/elasticsearch/Version.java | 12 +- .../elasticsearch/action/ActionModule.java | 49 +- .../action/DocWriteResponse.java | 4 +- ...ansportClusterAllocationExplainAction.java | 22 +- .../AddVotingConfigExclusionsRequest.java | 2 +- ...nsportAddVotingConfigExclusionsAction.java | 12 +- ...portClearVotingConfigExclusionsAction.java | 16 +- .../health/TransportClusterHealthAction.java | 12 +- .../TransportNodesHotThreadsAction.java | 4 +- .../cluster/node/info/NodesInfoRequest.java | 227 +- .../node/info/NodesInfoRequestBuilder.java | 29 +- .../node/info/TransportNodesInfoAction.java | 19 +- ...nsportNodesReloadSecureSettingsAction.java | 4 +- .../admin/cluster/node/stats/NodeStats.java | 25 +- .../cluster/node/stats/NodesStatsRequest.java | 223 +- .../node/stats/NodesStatsRequestBuilder.java | 43 +- .../node/stats/TransportNodesStatsAction.java | 24 +- .../node/tasks/cancel/CancelTasksRequest.java | 21 + .../cancel/CancelTasksRequestBuilder.java | 4 + .../cancel/TransportCancelTasksAction.java | 192 +- .../node/usage/TransportNodesUsageAction.java | 4 +- .../get/GetRepositoriesResponse.java | 16 +- .../get/TransportGetRepositoriesAction.java | 26 +- .../TransportVerifyRepositoryAction.java | 2 +- .../cluster/settings/SettingsUpdater.java | 28 +- .../TransportClusterUpdateSettingsAction.java | 10 +- .../create/TransportCreateSnapshotAction.java | 4 +- .../delete/TransportDeleteSnapshotAction.java | 4 +- .../get/TransportGetSnapshotsAction.java | 107 +- .../restore/RestoreSnapshotResponse.java | 2 +- .../status/SnapshotIndexShardStatus.java | 4 +- .../status/TransportNodesSnapshotsStatus.java | 4 +- .../TransportSnapshotsStatusAction.java | 100 +- .../cluster/state/ClusterStateRequest.java | 38 +- .../state/ClusterStateRequestBuilder.java | 17 +- .../state/TransportClusterStateAction.java | 36 +- .../admin/cluster/stats/AnalysisStats.java | 14 +- .../admin/cluster/stats/MappingStats.java | 12 +- .../stats/TransportClusterStatsAction.java | 8 +- .../alias/TransportIndicesAliasesAction.java | 22 +- .../indices/alias/get/GetAliasesResponse.java | 20 +- .../alias/get/TransportGetAliasesAction.java | 12 +- .../close/TransportCloseIndexAction.java | 6 +- .../CreateIndexClusterStateUpdateRequest.java | 4 +- .../create/TransportCreateIndexAction.java | 6 +- .../datastream/CreateDataStreamAction.java | 38 +- .../datastream/DeleteDataStreamAction.java | 36 +- .../datastream/GetDataStreamsAction.java | 36 +- .../delete/TransportDeleteIndexAction.java | 6 +- .../admin/indices/get/GetIndexResponse.java | 56 +- .../indices/get/TransportGetIndexAction.java | 18 +- .../mapping/get/GetFieldMappingsResponse.java | 44 +- .../mapping/get/GetMappingsResponse.java | 30 +- .../get/TransportGetFieldMappingsAction.java | 2 +- .../TransportGetFieldMappingsIndexAction.java | 14 +- .../get/TransportGetMappingsAction.java | 6 +- .../put/TransportPutMappingAction.java | 10 +- .../open/TransportOpenIndexAction.java | 6 +- .../rollover/MetadataRolloverService.java | 204 ++ .../rollover/TransportRolloverAction.java | 174 +- .../get/TransportGetSettingsAction.java | 10 +- .../put/TransportUpdateSettingsAction.java | 14 +- .../TransportIndicesShardStoresAction.java | 4 +- .../admin/indices/shrink/ResizeRequest.java | 4 +- .../indices/shrink/TransportResizeAction.java | 38 +- .../delete/DeleteIndexTemplateV2Action.java | 108 + ...ransportDeleteComponentTemplateAction.java | 6 +- .../TransportDeleteIndexTemplateAction.java | 12 +- .../TransportDeleteIndexTemplateV2Action.java | 79 + .../get/GetComponentTemplateAction.java | 39 +- .../get/GetIndexTemplateV2Action.java | 174 ++ .../get/GetIndexTemplatesResponse.java | 16 +- .../TransportGetComponentTemplateAction.java | 22 +- .../TransportGetIndexTemplateV2Action.java | 96 + .../get/TransportGetIndexTemplatesAction.java | 12 +- .../put/PutIndexTemplateV2Action.java | 181 ++ .../TransportPutComponentTemplateAction.java | 10 +- .../put/TransportPutIndexTemplateAction.java | 16 +- .../TransportPutIndexTemplateV2Action.java | 78 + .../upgrade/post/TransportUpgradeAction.java | 8 +- .../post/TransportUpgradeSettingsAction.java | 6 +- .../action/bulk/BulkProcessor.java | 3 +- .../org/elasticsearch/action/bulk/Retry.java | 3 +- .../action/bulk/TransportBulkAction.java | 108 +- .../action/bulk/TransportShardBulkAction.java | 10 +- .../explain/TransportExplainAction.java | 2 +- ...TransportFieldCapabilitiesIndexAction.java | 2 +- .../action/get/TransportGetAction.java | 4 +- .../action/get/TransportMultiGetAction.java | 4 +- .../action/index/IndexRequest.java | 10 +- .../ingest/PutPipelineTransportAction.java | 4 +- .../ingest/SimulatePipelineRequest.java | 16 +- .../ingest/WriteableIngestDocument.java | 26 +- .../action/main/TransportMainAction.java | 2 +- .../action/search/SearchRequest.java | 4 +- .../action/search/ShardSearchFailure.java | 4 +- .../action/support/ActiveShardCount.java | 12 +- .../support/nodes/TransportNodesAction.java | 2 +- .../replication/ReplicationResponse.java | 4 +- .../TransportBroadcastReplicationAction.java | 8 +- .../TransportReplicationAction.java | 28 +- .../TransportMultiTermVectorsAction.java | 6 +- .../TransportTermVectorsAction.java | 4 +- .../action/update/TransportUpdateAction.java | 10 +- .../bootstrap/BootstrapContext.java | 12 +- .../cluster/ClusterChangedEvent.java | 56 +- .../elasticsearch/cluster/ClusterModule.java | 99 +- .../elasticsearch/cluster/ClusterState.java | 190 +- .../cluster/InternalClusterInfoService.java | 2 +- ...aData.java => MergableCustomMetadata.java} | 8 +- .../cluster/SnapshotDeletionsInProgress.java | 3 + .../cluster/SnapshotsInProgress.java | 8 +- .../index/NodeMappingRefreshAction.java | 14 +- .../action/shard/ShardStateAction.java | 20 +- .../cluster/block/ClusterBlocks.java | 38 +- .../coordination/ClusterBootstrapService.java | 2 +- .../ClusterFormationFailureHelper.java | 3 +- ...etaData.java => CoordinationMetadata.java} | 31 +- .../coordination/CoordinationState.java | 30 +- .../cluster/coordination/Coordinator.java | 47 +- .../coordination/DetachClusterCommand.java | 16 +- .../ElasticsearchNodeCommand.java | 26 +- .../coordination/ElectionStrategy.java | 2 +- .../coordination/FollowersChecker.java | 10 - .../coordination/InMemoryPersistedState.java | 1 - .../cluster/coordination/JoinHelper.java | 18 +- .../coordination/JoinTaskExecutor.java | 44 +- .../NodeRemovalClusterStateTaskExecutor.java | 4 +- .../coordination/PreVoteCollector.java | 1 - .../cluster/coordination/Publication.java | 1 - .../cluster/coordination/Reconfigurator.java | 2 +- .../coordination/RemoveCustomsCommand.java | 12 +- .../coordination/RemoveSettingsCommand.java | 6 +- .../UnsafeBootstrapMasterCommand.java | 34 +- .../cluster/health/ClusterIndexHealth.java | 10 +- .../cluster/health/ClusterStateHealth.java | 8 +- .../cluster/metadata/AliasAction.java | 18 +- ...{AliasMetaData.java => AliasMetadata.java} | 62 +- .../cluster/metadata/AliasOrIndex.java | 198 -- .../cluster/metadata/AliasValidator.java | 16 +- .../cluster/metadata/AutoExpandReplicas.java | 30 +- .../cluster/metadata/ComponentTemplate.java | 2 +- .../metadata/ComponentTemplateMetadata.java | 16 +- .../cluster/metadata/DataStream.java | 15 +- .../cluster/metadata/DataStreamMetadata.java | 18 +- .../cluster/metadata/IndexAbstraction.java | 260 +++ .../cluster/metadata/IndexGraveyard.java | 14 +- ...{IndexMetaData.java => IndexMetadata.java} | 282 +-- .../metadata/IndexNameExpressionResolver.java | 178 +- ...taData.java => IndexTemplateMetadata.java} | 105 +- .../cluster/metadata/IndexTemplateV2.java | 6 +- .../metadata/IndexTemplateV2Metadata.java | 16 +- .../cluster/metadata/Manifest.java | 4 +- ...pingMetaData.java => MappingMetadata.java} | 18 +- .../MetaDataIndexTemplateService.java | 586 ------ .../metadata/{MetaData.java => Metadata.java} | 501 ++--- ...e.java => MetadataCreateIndexService.java} | 493 +++-- ...e.java => MetadataDeleteIndexService.java} | 20 +- ....java => MetadataIndexAliasesService.java} | 30 +- ...ce.java => MetadataIndexStateService.java} | 98 +- .../MetadataIndexTemplateService.java | 1020 +++++++++ ....java => MetadataIndexUpgradeService.java} | 64 +- ...rvice.java => MetadataMappingService.java} | 62 +- ...ava => MetadataUpdateSettingsService.java} | 106 +- ...etaData.java => RepositoriesMetadata.java} | 56 +- ...yMetaData.java => RepositoryMetadata.java} | 18 +- .../cluster/metadata/Template.java | 32 +- .../metadata/TemplateUpgradeService.java | 36 +- .../cluster/node/DiscoveryNode.java | 42 +- .../cluster/node/DiscoveryNodeRole.java | 24 +- .../cluster/routing/IndexRoutingTable.java | 70 +- .../cluster/routing/OperationRouting.java | 36 +- .../cluster/routing/RecoverySource.java | 6 +- .../cluster/routing/RoutingNodes.java | 20 +- .../cluster/routing/RoutingTable.java | 62 +- .../cluster/routing/ShardRouting.java | 4 +- .../cluster/routing/UnassignedInfo.java | 6 +- .../routing/allocation/AllocationService.java | 221 +- .../allocation/DiskThresholdMonitor.java | 10 +- .../allocation/ExistingShardsAllocator.java | 123 ++ ...Updater.java => IndexMetadataUpdater.java} | 104 +- .../routing/allocation/RoutingAllocation.java | 22 +- .../allocator/BalancedShardsAllocator.java | 20 +- .../AbstractAllocateAllocationCommand.java | 2 +- .../command/CancelAllocationCommand.java | 12 +- .../allocation/decider/AllocationDecider.java | 6 +- .../decider/AllocationDeciders.java | 10 +- .../decider/AwarenessAllocationDecider.java | 6 +- .../decider/DiskThresholdDecider.java | 32 +- .../decider/EnableAllocationDecider.java | 16 +- .../decider/FilterAllocationDecider.java | 24 +- .../decider/MaxRetryAllocationDecider.java | 6 +- ...alanceOnlyWhenActiveAllocationDecider.java | 2 +- .../decider/ResizeAllocationDecider.java | 18 +- .../decider/ShardsLimitAllocationDecider.java | 4 +- .../service/ClusterApplierService.java | 4 +- .../cluster/service/ClusterService.java | 4 +- .../cluster/service/MasterService.java | 6 +- .../common/blobstore/BlobContainer.java | 42 +- .../{BlobMetaData.java => BlobMetadata.java} | 2 +- .../common/blobstore/fs/FsBlobContainer.java | 36 +- ...obMetaData.java => PlainBlobMetadata.java} | 6 +- .../common/bytes/ByteBufferReference.java | 19 + .../common/bytes/BytesReference.java | 9 +- .../common/bytes/CompositeBytesReference.java | 28 + .../bytes/ReleasableBytesReference.java | 27 +- .../common/compress/Compressor.java | 2 + .../common/compress/DeflateCompressor.java | 5 + .../org/elasticsearch/common/io/Streams.java | 103 + .../common/io/stream/StreamOutput.java | 16 +- .../common/lease/Releasables.java | 2 +- .../common/logging/ClusterIdConverter.java | 59 + .../logging/CustomMapFieldsConverter.java | 18 +- .../common/logging/DeprecatedMessage.java | 13 +- .../common/logging/ECSJsonLayout.java | 85 + .../common/logging/ESJsonLayout.java | 30 +- .../common/logging/ESLogMessage.java | 36 +- .../logging/NodeAndClusterIdConverter.java | 30 +- .../NodeAndClusterIdStateListener.java | 15 +- .../common/logging/NodeIdConverter.java | 58 + .../logging/NodeNamePatternConverter.java | 12 +- .../common/settings/ClusterSettings.java | 13 +- .../settings/ConsistentSettingsService.java | 12 +- .../common/settings/IndexScopedSettings.java | 70 +- .../common/unit/ByteSizeValue.java | 3 +- .../common/util/concurrent/ThreadContext.java | 9 +- .../common/util/iterable/Iterables.java | 4 + .../common/xcontent/XContentHelper.java | 19 + .../org/elasticsearch/env/Environment.java | 20 +- .../elasticsearch/env/NodeEnvironment.java | 95 +- .../{NodeMetaData.java => NodeMetadata.java} | 34 +- .../env/NodeRepurposeCommand.java | 44 +- .../env/OverrideNodeVersionCommand.java | 14 +- .../gateway/BaseGatewayShardAllocator.java | 45 +- .../gateway/ClusterStateUpdaters.java | 58 +- .../gateway/DanglingIndicesState.java | 64 +- .../org/elasticsearch/gateway/Gateway.java | 48 +- .../gateway/GatewayAllocator.java | 95 +- .../gateway/GatewayMetaState.java | 98 +- .../elasticsearch/gateway/GatewayService.java | 2 +- .../IncrementalClusterStateWriter.java | 100 +- .../gateway/LocalAllocateDangledIndices.java | 70 +- .../gateway/MetaStateService.java | 116 +- ...teFormat.java => MetadataStateFormat.java} | 12 +- .../gateway/PersistedClusterStateService.java | 208 +- .../gateway/PrimaryShardAllocator.java | 6 +- .../gateway/PriorityComparator.java | 10 +- .../gateway/ReplicaShardAllocator.java | 84 +- .../TransportNodesListGatewayMetaState.java | 24 +- ...ransportNodesListGatewayStartedShards.java | 34 +- .../index/CompositeIndexEventListener.java | 13 + .../org/elasticsearch/index/IndexModule.java | 37 +- .../org/elasticsearch/index/IndexService.java | 48 +- .../elasticsearch/index/IndexSettings.java | 46 +- .../elasticsearch/index/IndexingSlowLog.java | 35 +- .../elasticsearch/index/SearchSlowLog.java | 42 +- .../index/analysis/AnalysisRegistry.java | 11 +- .../index/analysis/LowercaseNormalizer.java | 42 + .../analysis/LowercaseNormalizerProvider.java | 44 + .../index/engine/CombinedDeletionPolicy.java | 10 +- .../index/get/ShardGetService.java | 8 +- .../index/mapper/BooleanFieldMapper.java | 2 +- .../index/mapper/CompletionFieldMapper.java | 38 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/IdFieldMapper.java | 2 + .../index/mapper/IpFieldMapper.java | 2 +- .../index/mapper/MapperService.java | 54 +- .../elasticsearch/index/mapper/Mapping.java | 8 +- .../index/mapper/ObjectMapper.java | 17 +- .../index/query/QueryShardContext.java | 17 +- .../index/reindex/RetryListener.java | 3 +- .../reindex/WorkerBulkByScrollTaskState.java | 8 +- .../index/seqno/ReplicationTracker.java | 4 +- .../index/seqno/RetentionLeases.java | 4 +- .../index/shard/IndexEventListener.java | 11 + .../elasticsearch/index/shard/IndexShard.java | 58 +- .../index/shard/IndexingStats.java | 4 +- .../index/shard/LocalShardSnapshot.java | 6 +- .../RemoveCorruptedShardDataCommand.java | 36 +- .../elasticsearch/index/shard/ShardId.java | 4 +- .../elasticsearch/index/shard/ShardPath.java | 10 +- .../index/shard/ShardSplittingQuery.java | 22 +- ...eMetaData.java => ShardStateMetadata.java} | 30 +- .../index/shard/StoreRecovery.java | 28 +- .../BlobStoreIndexShardSnapshot.java | 20 +- .../org/elasticsearch/index/store/Store.java | 100 +- ...leMetaData.java => StoreFileMetadata.java} | 10 +- .../index/translog/Translog.java | 58 +- .../translog/TruncateTranslogAction.java | 8 +- .../elasticsearch/indices/IndicesModule.java | 5 +- .../indices/IndicesRequestCache.java | 2 +- .../elasticsearch/indices/IndicesService.java | 198 +- .../indices/analysis/AnalysisModule.java | 13 +- .../cluster/IndicesClusterStateService.java | 82 +- .../indices/mapper/MapperRegistry.java | 2 +- .../indices/recovery/MultiFileTransfer.java | 28 +- .../indices/recovery/MultiFileWriter.java | 24 +- .../recovery/PeerRecoveryTargetService.java | 12 +- .../recovery/RecoveryFileChunkRequest.java | 26 +- .../recovery/RecoverySourceHandler.java | 38 +- .../indices/recovery/RecoveryTarget.java | 10 +- .../recovery/RecoveryTargetHandler.java | 8 +- .../recovery/RemoteRecoveryTargetHandler.java | 10 +- .../indices/store/IndicesStore.java | 10 +- ...TransportNodesListShardStoreMetadata.java} | 104 +- .../elasticsearch/ingest/IngestDocument.java | 30 +- .../elasticsearch/ingest/IngestMetadata.java | 16 +- .../elasticsearch/ingest/IngestService.java | 28 +- .../org/elasticsearch/monitor/fs/FsInfo.java | 6 + .../monitor/jvm/JvmGcMonitorService.java | 21 +- .../java/org/elasticsearch/node/Node.java | 86 +- .../org/elasticsearch/node/NodeService.java | 5 +- .../persistent/AllocatedPersistentTask.java | 8 +- .../persistent/PersistentTaskResponse.java | 2 +- .../PersistentTasksClusterService.java | 54 +- ...ava => PersistentTasksCustomMetadata.java} | 48 +- .../persistent/PersistentTasksExecutor.java | 6 +- .../PersistentTasksNodeService.java | 6 +- .../persistent/PersistentTasksService.java | 12 +- .../persistent/package-info.java | 4 +- .../elasticsearch/plugins/ClusterPlugin.java | 20 +- ...ataUpgrader.java => MetadataUpgrader.java} | 18 +- .../org/elasticsearch/plugins/Plugin.java | 8 +- .../plugins/RepositoryPlugin.java | 10 + .../elasticsearch/plugins/SearchPlugin.java | 27 + .../plugins/SystemIndexPlugin.java | 4 +- .../repositories/FilterRepository.java | 23 +- .../repositories/RepositoriesModule.java | 2 + .../repositories/RepositoriesService.java | 188 +- .../repositories/Repository.java | 28 +- .../blobstore/BlobStoreRepository.java | 172 +- .../blobstore/ChecksumBlobStoreFormat.java | 6 +- .../blobstore/FileRestoreContext.java | 20 +- .../repositories/blobstore/package-info.java | 18 +- .../repositories/fs/FsRepository.java | 4 +- .../elasticsearch/rest/BaseRestHandler.java | 53 + .../admin/cluster/RestCancelTasksAction.java | 1 + .../cluster/RestClusterGetSettingsAction.java | 6 +- .../admin/cluster/RestClusterStateAction.java | 13 +- .../admin/cluster/RestNodesInfoAction.java | 25 +- .../admin/cluster/RestNodesStatsAction.java | 25 +- .../RestDeleteIndexTemplateV2Action.java | 53 + .../admin/indices/RestGetAliasesAction.java | 22 +- .../RestGetComponentTemplateAction.java | 6 +- .../indices/RestGetDataStreamsAction.java | 4 +- .../indices/RestGetFieldMappingAction.java | 4 +- .../indices/RestGetIndexTemplateV2Action.java | 78 + .../indices/RestPutIndexTemplateV2Action.java | 60 + .../rest/action/cat/RestAliasAction.java | 16 +- .../rest/action/cat/RestAllocationAction.java | 3 +- .../rest/action/cat/RestIndicesAction.java | 22 +- .../rest/action/cat/RestNodeAttrsAction.java | 3 +- .../rest/action/cat/RestNodesAction.java | 14 +- .../rest/action/cat/RestPluginsAction.java | 3 +- .../action/cat/RestRepositoriesAction.java | 8 +- .../rest/action/cat/RestShardsAction.java | 2 +- .../rest/action/cat/RestTasksAction.java | 2 +- .../rest/action/cat/RestTemplatesAction.java | 12 +- .../rest/action/cat/RestThreadPoolAction.java | 6 +- .../rest/action/document/RestIndexAction.java | 13 +- .../rest/action/search/RestSearchAction.java | 2 +- .../org/elasticsearch/script/ScriptCache.java | 7 +- .../script/ScriptCacheStats.java | 147 ++ ...criptMetaData.java => ScriptMetadata.java} | 64 +- .../elasticsearch/script/ScriptService.java | 45 +- .../script/StoredScriptSource.java | 4 +- .../elasticsearch/script/UpdateScript.java | 5 +- .../org/elasticsearch/search/SearchHit.java | 187 +- .../elasticsearch/search/SearchModule.java | 110 +- .../elasticsearch/search/SearchService.java | 250 +-- .../AbstractAggregationBuilder.java | 30 +- .../search/aggregations/Aggregation.java | 2 +- .../aggregations/AggregationBuilder.java | 25 +- .../aggregations/AggregationBuilders.java | 10 +- .../search/aggregations/AggregationPhase.java | 16 +- .../search/aggregations/AggregatorBase.java | 19 +- .../aggregations/AggregatorFactories.java | 15 +- .../aggregations/AggregatorFactory.java | 13 +- .../aggregations/BaseAggregationBuilder.java | 4 +- .../aggregations/InternalAggregation.java | 93 +- .../aggregations/InternalAggregations.java | 106 +- .../InternalMultiBucketAggregation.java | 15 +- .../aggregations/NonCollectingAggregator.java | 10 +- .../aggregations/ParsedAggregation.java | 4 +- .../PipelineAggregationBuilder.java | 17 +- .../bucket/BucketsAggregator.java | 6 +- .../bucket/DeferableBucketAggregator.java | 5 +- .../InternalSingleBucketAggregation.java | 11 +- .../AdjacencyMatrixAggregationBuilder.java | 19 +- .../adjacency/AdjacencyMatrixAggregator.java | 12 +- .../AdjacencyMatrixAggregatorFactory.java | 11 +- .../adjacency/InternalAdjacencyMatrix.java | 11 +- .../CompositeAggregationBuilder.java | 20 +- .../CompositeAggregationFactory.java | 11 +- .../bucket/composite/CompositeAggregator.java | 9 +- .../CompositeValuesSourceBuilder.java | 7 +- .../CompositeValuesSourceParserHelper.java | 13 +- .../DateHistogramValuesSourceBuilder.java | 4 +- .../GeoTileGridValuesSourceBuilder.java | 4 +- .../HistogramValuesSourceBuilder.java | 4 +- .../bucket/composite/InternalComposite.java | 9 +- .../composite/TermsValuesSourceBuilder.java | 4 +- .../filter/FilterAggregationBuilder.java | 15 +- .../bucket/filter/FilterAggregator.java | 13 +- .../filter/FilterAggregatorFactory.java | 13 +- .../filter/FiltersAggregationBuilder.java | 19 +- .../bucket/filter/FiltersAggregator.java | 10 +- .../filter/FiltersAggregatorFactory.java | 12 +- .../bucket/filter/InternalFilter.java | 9 +- .../bucket/filter/InternalFilters.java | 11 +- .../geogrid/GeoGridAggregationBuilder.java | 52 +- .../bucket/geogrid/GeoGridAggregator.java | 12 +- .../GeoHashGridAggregationBuilder.java | 28 +- .../bucket/geogrid/GeoHashGridAggregator.java | 13 +- .../geogrid/GeoHashGridAggregatorFactory.java | 53 +- .../GeoTileGridAggregationBuilder.java | 22 +- .../bucket/geogrid/GeoTileGridAggregator.java | 13 +- .../geogrid/GeoTileGridAggregatorFactory.java | 50 +- .../bucket/geogrid/InternalGeoGrid.java | 11 +- .../bucket/geogrid/InternalGeoHashGrid.java | 12 +- .../bucket/geogrid/InternalGeoTileGrid.java | 12 +- .../global/GlobalAggregationBuilder.java | 15 +- .../bucket/global/GlobalAggregator.java | 9 +- .../global/GlobalAggregatorFactory.java | 11 +- .../bucket/global/InternalGlobal.java | 9 +- .../AutoDateHistogramAggregationBuilder.java | 96 +- .../AutoDateHistogramAggregator.java | 11 +- .../AutoDateHistogramAggregatorFactory.java | 32 +- .../DateHistogramAggregationBuilder.java | 74 +- .../DateHistogramAggregationSupplier.java | 50 + .../histogram/DateHistogramAggregator.java | 11 +- .../DateHistogramAggregatorFactory.java | 115 +- .../DateRangeHistogramAggregator.java | 12 +- .../HistogramAggregationBuilder.java | 55 +- .../histogram/HistogramAggregatorFactory.java | 87 +- .../HistogramAggregatorSupplier.java | 38 + .../histogram/InternalAutoDateHistogram.java | 11 +- .../histogram/InternalDateHistogram.java | 15 +- .../bucket/histogram/InternalHistogram.java | 14 +- .../histogram/NumericHistogramAggregator.java | 16 +- .../histogram/RangeHistogramAggregator.java | 14 +- .../bucket/missing/InternalMissing.java | 9 +- .../missing/MissingAggregationBuilder.java | 50 +- .../bucket/missing/MissingAggregator.java | 12 +- .../missing/MissingAggregatorFactory.java | 50 +- .../missing/MissingAggregatorSupplier.java | 41 + .../bucket/nested/InternalNested.java | 9 +- .../bucket/nested/InternalReverseNested.java | 9 +- .../nested/NestedAggregationBuilder.java | 17 +- .../bucket/nested/NestedAggregator.java | 12 +- .../nested/NestedAggregatorFactory.java | 20 +- .../ReverseNestedAggregationBuilder.java | 13 +- .../nested/ReverseNestedAggregator.java | 11 +- .../ReverseNestedAggregatorFactory.java | 20 +- .../range/AbstractRangeAggregatorFactory.java | 72 +- .../bucket/range/AbstractRangeBuilder.java | 24 +- .../bucket/range/BinaryRangeAggregator.java | 10 +- .../range/BinaryRangeAggregatorFactory.java | 29 +- .../range/DateRangeAggregationBuilder.java | 29 +- .../range/DateRangeAggregatorFactory.java | 20 +- .../range/GeoDistanceAggregationBuilder.java | 43 +- .../GeoDistanceRangeAggregatorFactory.java | 34 +- .../bucket/range/InternalBinaryRange.java | 10 +- .../bucket/range/InternalDateRange.java | 13 +- .../bucket/range/InternalGeoDistance.java | 14 +- .../bucket/range/InternalRange.java | 17 +- .../range/IpRangeAggregationBuilder.java | 57 +- .../bucket/range/RangeAggregationBuilder.java | 22 +- .../bucket/range/RangeAggregator.java | 15 +- .../bucket/range/RangeAggregatorFactory.java | 16 +- .../bucket/range/RangeAggregatorSupplier.java | 42 + .../DiversifiedAggregationBuilder.java | 39 +- .../sampler/DiversifiedAggregatorFactory.java | 25 +- ...DiversifiedBytesHashSamplerAggregator.java | 6 +- .../DiversifiedMapSamplerAggregator.java | 6 +- .../DiversifiedNumericSamplerAggregator.java | 6 +- .../DiversifiedOrdinalsSamplerAggregator.java | 6 +- .../bucket/sampler/InternalSampler.java | 9 +- .../sampler/SamplerAggregationBuilder.java | 15 +- .../bucket/sampler/SamplerAggregator.java | 30 +- .../sampler/SamplerAggregatorFactory.java | 11 +- .../bucket/sampler/UnmappedSampler.java | 7 +- ...balOrdinalsSignificantTermsAggregator.java | 11 +- .../InternalMappedSignificantTerms.java | 9 +- .../significant/InternalSignificantTerms.java | 6 +- .../significant/SignificantLongTerms.java | 11 +- .../SignificantLongTermsAggregator.java | 11 +- .../significant/SignificantStringTerms.java | 12 +- .../SignificantStringTermsAggregator.java | 12 +- .../SignificantTermsAggregationBuilder.java | 52 +- .../SignificantTermsAggregatorFactory.java | 215 +- .../SignificantTermsAggregatorSupplier.java | 47 + .../SignificantTextAggregationBuilder.java | 15 +- .../SignificantTextAggregator.java | 13 +- .../SignificantTextAggregatorFactory.java | 13 +- .../significant/UnmappedSignificantTerms.java | 10 +- .../terms/AbstractRareTermsAggregator.java | 7 +- .../terms/AbstractStringTermsAggregator.java | 10 +- .../bucket/terms/DoubleTerms.java | 13 +- .../bucket/terms/DoubleTermsAggregator.java | 9 +- .../GlobalOrdinalsStringTermsAggregator.java | 15 +- .../bucket/terms/InternalMappedRareTerms.java | 6 +- .../bucket/terms/InternalMappedTerms.java | 5 +- .../bucket/terms/InternalRareTerms.java | 6 +- .../bucket/terms/InternalTerms.java | 6 +- .../bucket/terms/LongRareTerms.java | 11 +- .../bucket/terms/LongRareTermsAggregator.java | 11 +- .../aggregations/bucket/terms/LongTerms.java | 17 +- .../bucket/terms/LongTermsAggregator.java | 15 +- .../terms/RareTermsAggregationBuilder.java | 48 +- .../terms/RareTermsAggregatorFactory.java | 148 +- .../terms/RareTermsAggregatorSupplier.java | 42 + .../bucket/terms/StringRareTerms.java | 11 +- .../terms/StringRareTermsAggregator.java | 11 +- .../bucket/terms/StringTerms.java | 13 +- .../bucket/terms/StringTermsAggregator.java | 9 +- .../bucket/terms/TermsAggregationBuilder.java | 51 +- .../bucket/terms/TermsAggregator.java | 6 +- .../bucket/terms/TermsAggregatorFactory.java | 260 ++- .../bucket/terms/TermsAggregatorSupplier.java | 46 + .../bucket/terms/UnmappedRareTerms.java | 9 +- .../bucket/terms/UnmappedTerms.java | 10 +- .../AbstractHDRPercentilesAggregator.java | 48 +- .../AbstractInternalHDRPercentiles.java | 10 +- .../AbstractInternalTDigestPercentiles.java | 10 +- ...AbstractPercentilesAggregationBuilder.java | 17 +- .../AbstractTDigestPercentilesAggregator.java | 37 +- .../metrics/AvgAggregationBuilder.java | 32 +- .../aggregations/metrics/AvgAggregator.java | 10 +- .../metrics/AvgAggregatorFactory.java | 50 +- .../CardinalityAggregationBuilder.java | 35 +- .../metrics/CardinalityAggregator.java | 11 +- .../metrics/CardinalityAggregatorFactory.java | 49 +- .../CardinalityAggregatorSupplier.java | 38 + .../ExtendedStatsAggregationBuilder.java | 31 +- .../metrics/ExtendedStatsAggregator.java | 13 +- .../ExtendedStatsAggregatorFactory.java | 47 +- .../ExtendedStatsAggregatorProvider.java | 39 + .../metrics/GeoBoundsAggregationBuilder.java | 49 +- .../metrics/GeoBoundsAggregator.java | 13 +- .../metrics/GeoBoundsAggregatorFactory.java | 42 +- .../metrics/GeoBoundsAggregatorSupplier.java | 35 + .../GeoCentroidAggregationBuilder.java | 37 +- .../metrics/GeoCentroidAggregator.java | 11 +- .../metrics/GeoCentroidAggregatorFactory.java | 39 +- .../GeoCentroidAggregatorSupplier.java | 35 + .../metrics/GeoGridAggregatorSupplier.java | 39 + .../metrics/HDRPercentileRanksAggregator.java | 12 +- .../metrics/HDRPercentilesAggregator.java | 14 +- .../aggregations/metrics/InternalAvg.java | 8 +- .../metrics/InternalCardinality.java | 8 +- .../metrics/InternalExtendedStats.java | 7 +- .../metrics/InternalGeoBounds.java | 9 +- .../metrics/InternalGeoCentroid.java | 8 +- .../metrics/InternalHDRPercentileRanks.java | 14 +- .../metrics/InternalHDRPercentiles.java | 12 +- .../aggregations/metrics/InternalMax.java | 8 +- .../InternalMedianAbsoluteDeviation.java | 12 +- .../aggregations/metrics/InternalMin.java | 8 +- .../InternalNumericMetricsAggregation.java | 13 +- .../metrics/InternalScriptedMetric.java | 56 +- .../aggregations/metrics/InternalStats.java | 7 +- .../aggregations/metrics/InternalSum.java | 8 +- .../InternalTDigestPercentileRanks.java | 14 +- .../metrics/InternalTDigestPercentiles.java | 12 +- .../aggregations/metrics/InternalTopHits.java | 7 +- .../metrics/InternalValueCount.java | 8 +- .../metrics/InternalWeightedAvg.java | 9 +- .../metrics/MaxAggregationBuilder.java | 32 +- .../aggregations/metrics/MaxAggregator.java | 13 +- .../metrics/MaxAggregatorFactory.java | 49 +- ...anAbsoluteDeviationAggregationBuilder.java | 38 +- .../MedianAbsoluteDeviationAggregator.java | 11 +- ...ianAbsoluteDeviationAggregatorFactory.java | 79 +- ...anAbsoluteDeviationAggregatorSupplier.java | 38 + .../metrics/MetricAggregatorSupplier.java | 37 + .../metrics/MetricsAggregator.java | 8 +- .../metrics/MinAggregationBuilder.java | 32 +- .../aggregations/metrics/MinAggregator.java | 15 +- .../metrics/MinAggregatorFactory.java | 49 +- .../metrics/MinMaxAggregatorSupplier.java | 37 + .../metrics/NumericMetricsAggregator.java | 16 +- .../PercentileRanksAggregationBuilder.java | 27 +- .../PercentileRanksAggregatorFactory.java | 50 +- .../PercentilesAggregationBuilder.java | 27 +- .../metrics/PercentilesAggregatorFactory.java | 50 +- .../PercentilesAggregatorSupplier.java | 40 + .../metrics/PercentilesConfig.java | 49 +- .../ScriptedMetricAggregationBuilder.java | 15 +- .../metrics/ScriptedMetricAggregator.java | 12 +- .../ScriptedMetricAggregatorFactory.java | 15 +- .../metrics/StatsAggregationBuilder.java | 34 +- .../aggregations/metrics/StatsAggregator.java | 12 +- .../metrics/StatsAggregatorFactory.java | 50 +- .../metrics/SumAggregationBuilder.java | 32 +- .../aggregations/metrics/SumAggregator.java | 10 +- .../metrics/SumAggregatorFactory.java | 49 +- .../TDigestPercentileRanksAggregator.java | 12 +- .../metrics/TDigestPercentilesAggregator.java | 12 +- .../metrics/TopHitsAggregationBuilder.java | 15 +- .../metrics/TopHitsAggregator.java | 11 +- .../metrics/TopHitsAggregatorFactory.java | 11 +- .../metrics/ValueCountAggregationBuilder.java | 35 +- .../metrics/ValueCountAggregator.java | 12 +- .../metrics/ValueCountAggregatorFactory.java | 45 +- .../metrics/ValueCountAggregatorSupplier.java | 35 + .../WeightedAvgAggregationBuilder.java | 39 +- .../metrics/WeightedAvgAggregator.java | 11 +- .../metrics/WeightedAvgAggregatorFactory.java | 25 +- .../AbstractPipelineAggregationBuilder.java | 22 +- .../AvgBucketPipelineAggregationBuilder.java | 4 +- .../pipeline/AvgBucketPipelineAggregator.java | 9 +- ...cketMetricsPipelineAggregationBuilder.java | 5 +- .../BucketMetricsPipelineAggregator.java | 14 +- ...ucketScriptPipelineAggregationBuilder.java | 6 +- .../BucketScriptPipelineAggregator.java | 3 +- ...ketSelectorPipelineAggregationBuilder.java | 6 +- .../BucketSortPipelineAggregationBuilder.java | 5 +- ...mulativeSumPipelineAggregationBuilder.java | 4 +- .../CumulativeSumPipelineAggregator.java | 2 +- .../DerivativePipelineAggregationBuilder.java | 4 +- .../DerivativePipelineAggregator.java | 2 +- ...StatsBucketPipelineAggregationBuilder.java | 4 +- ...ExtendedStatsBucketPipelineAggregator.java | 9 +- .../pipeline/InternalBucketMetricValue.java | 5 +- .../pipeline/InternalDerivative.java | 5 +- .../pipeline/InternalExtendedStatsBucket.java | 5 +- .../pipeline/InternalPercentilesBucket.java | 5 +- .../pipeline/InternalSimpleValue.java | 5 +- .../pipeline/InternalStatsBucket.java | 4 +- .../MaxBucketPipelineAggregationBuilder.java | 4 +- .../pipeline/MaxBucketPipelineAggregator.java | 9 +- .../MinBucketPipelineAggregationBuilder.java | 4 +- .../pipeline/MinBucketPipelineAggregator.java | 9 +- .../MovFnPipelineAggregationBuilder.java | 4 +- .../pipeline/MovFnPipelineAggregator.java | 2 +- ...tilesBucketPipelineAggregationBuilder.java | 4 +- .../PercentilesBucketPipelineAggregator.java | 9 +- .../pipeline/PipelineAggregator.java | 14 +- .../SerialDiffPipelineAggregationBuilder.java | 4 +- .../SerialDiffPipelineAggregator.java | 2 +- .../pipeline/SiblingPipelineAggregator.java | 4 +- ...StatsBucketPipelineAggregationBuilder.java | 4 +- .../StatsBucketPipelineAggregator.java | 9 +- .../SumBucketPipelineAggregationBuilder.java | 4 +- .../pipeline/SumBucketPipelineAggregator.java | 9 +- .../support/AggregatorSupplier.java | 41 + .../support/CoreValuesSourceType.java | 166 +- .../support/MultiValuesSource.java | 12 +- .../MultiValuesSourceAggregationBuilder.java | 105 +- .../MultiValuesSourceAggregatorFactory.java | 28 +- .../support/MultiValuesSourceParseHelper.java | 18 +- .../aggregations/support/ValueType.java | 61 +- .../aggregations/support/ValuesSource.java | 45 +- .../ValuesSourceAggregationBuilder.java | 219 +- .../ValuesSourceAggregatorFactory.java | 44 +- .../support/ValuesSourceConfig.java | 348 ++-- .../support/ValuesSourceParserHelper.java | 112 - .../support/ValuesSourceRegistry.java | 193 ++ .../support/ValuesSourceType.java | 40 +- .../aggregations/support/package-info.java | 77 + .../search/fetch/FetchPhase.java | 16 +- .../fetch/subphase/FetchDocValuesPhase.java | 2 +- .../fetch/subphase/ScriptFieldsPhase.java | 3 +- .../search/internal/ShardSearchRequest.java | 37 +- .../search/query/QuerySearchResult.java | 24 +- .../completion/CompletionSuggestion.java | 6 +- .../suggest/phrase/PhraseSuggestion.java | 6 +- .../search/suggest/term/TermSuggestion.java | 6 +- .../snapshots/RestoreService.java | 197 +- .../elasticsearch/snapshots/SnapshotInfo.java | 13 +- .../snapshots/SnapshotShardFailure.java | 4 +- .../snapshots/SnapshotsService.java | 763 +++---- .../elasticsearch/tasks/CancellableTask.java | 17 +- .../org/elasticsearch/tasks/TaskManager.java | 217 +- .../tasks/TaskResultsService.java | 22 +- .../elasticsearch/threadpool/Scheduler.java | 12 +- .../elasticsearch/threadpool/ThreadPool.java | 10 +- .../org/elasticsearch/transport/Header.java | 111 + .../transport/InboundAggregator.java | 246 +++ .../transport/InboundDecoder.java | 213 ++ .../transport/InboundHandler.java | 176 +- .../transport/InboundMessage.java | 184 +- .../transport/InboundPipeline.java | 185 ++ .../transport/OutboundHandler.java | 13 +- .../transport/RemoteClusterService.java | 26 +- .../elasticsearch/transport/StatsTracker.java | 64 + .../elasticsearch/transport/TcpHeader.java | 12 +- .../elasticsearch/transport/TcpTransport.java | 108 +- .../transport/TcpTransportChannel.java | 18 +- .../elasticsearch/transport/Transport.java | 2 - .../transport/TransportDecompressor.java | 148 ++ .../transport/TransportHandshaker.java | 88 +- .../transport/TransportLogger.java | 74 +- .../transport/TransportService.java | 39 +- .../org/elasticsearch/usage/UsageService.java | 31 +- .../elasticsearch/ExceptionsHelperTests.java | 4 +- .../java/org/elasticsearch/VersionTests.java | 8 +- .../action/ActionModuleTests.java | 9 +- .../ClusterAllocationExplainActionTests.java | 5 +- .../ClusterAllocationExplainIT.java | 18 +- ...AddVotingConfigExclusionsRequestTests.java | 16 +- ...tAddVotingConfigExclusionsActionTests.java | 54 +- ...learVotingConfigExclusionsActionTests.java | 10 +- .../health/ClusterHealthResponsesTests.java | 4 +- .../TransportClusterHealthActionTests.java | 10 +- .../node/info/NodesInfoRequestTests.java | 128 +- .../cluster/node/stats/NodeStatsTests.java | 45 +- .../node/stats/NodesStatsRequestTests.java | 126 +- .../node/tasks/CancellableTasksIT.java | 479 +++++ .../node/tasks/CancellableTasksTests.java | 182 +- .../node/tasks/TaskManagerTestCase.java | 4 +- .../cluster/node/tasks/TestTaskPlugin.java | 3 +- .../node/tasks/TransportTasksActionTests.java | 8 +- .../repositories/RepositoryBlocksIT.java | 6 +- .../reroute/ClusterRerouteResponseTests.java | 38 +- .../cluster/reroute/ClusterRerouteTests.java | 14 +- .../settings/SettingsUpdaterTests.java | 138 +- .../cluster/snapshots/SnapshotBlocksIT.java | 8 +- .../status/SnapshotIndexShardStatusTests.java | 4 +- .../cluster/state/ClusterStateApiTests.java | 12 +- .../state/ClusterStateRequestTests.java | 14 +- ...ansportClusterStateActionDisruptionIT.java | 26 +- .../admin/cluster/stats/ClusterStatsIT.java | 16 +- .../cluster/tasks/PendingTasksBlocksIT.java | 10 +- .../indices/TransportAnalyzeActionTests.java | 6 +- .../ValidateIndicesAliasesRequestIT.java | 10 +- .../alias/get/GetAliasesResponseTests.java | 38 +- .../get/TransportGetAliasesActionTests.java | 16 +- .../clear/ClearIndicesCacheBlocksIT.java | 10 +- ...portVerifyShardBeforeCloseActionTests.java | 12 +- .../admin/indices/create/CreateIndexIT.java | 46 +- .../admin/indices/create/ShrinkIndexIT.java | 36 +- .../admin/indices/create/SplitIndexIT.java | 18 +- .../CreateDataStreamRequestTests.java | 62 +- .../DeleteDataStreamRequestTests.java | 104 +- .../GetDataStreamsRequestTests.java | 72 +- .../GetDataStreamsResponseTests.java | 5 +- .../indices/delete/DeleteIndexBlocksIT.java | 26 +- .../admin/indices/flush/FlushBlocksIT.java | 10 +- .../forcemerge/ForceMergeBlocksIT.java | 10 +- .../indices/forcemerge/ForceMergeIT.java | 6 +- .../action/admin/indices/get/GetIndexIT.java | 34 +- .../indices/get/GetIndexResponseTests.java | 16 +- .../get/GetFieldMappingsResponseTests.java | 28 +- .../mapping/get/GetMappingsResponseTests.java | 12 +- .../indices/refresh/RefreshBlocksIT.java | 10 +- .../MetadataRolloverServiceTests.java | 480 +++++ .../admin/indices/rollover/RolloverIT.java | 54 +- .../TransportRolloverActionTests.java | 326 +-- .../segments/IndicesSegmentsBlocksIT.java | 10 +- .../shards/IndicesShardStoreRequestIT.java | 12 +- .../indices/shrink/ResizeRequestTests.java | 2 +- .../shrink/TransportResizeActionTests.java | 18 +- .../indices/stats/IndicesStatsBlocksIT.java | 16 +- .../DeleteIndexTemplateV2RequestTests.java | 42 + .../get/GetIndexTemplateV2RequestTests.java | 42 + .../get/GetIndexTemplateV2ResponseTests.java | 54 + .../get/GetIndexTemplatesResponseTests.java | 10 +- .../put/PutIndexTemplateV2RequestTests.java | 47 + .../action/bulk/BulkIntegrationIT.java | 4 +- .../action/bulk/BulkProcessorIT.java | 4 +- .../action/bulk/BulkWithUpdatesIT.java | 6 +- ...ActionIndicesThatCannotBeCreatedTests.java | 4 +- .../bulk/TransportBulkActionIngestTests.java | 72 +- .../action/bulk/TransportBulkActionTests.java | 60 +- .../bulk/TransportShardBulkActionTests.java | 16 +- .../action/delete/DeleteResponseTests.java | 2 +- .../get/TransportMultiGetActionTests.java | 14 +- .../action/index/IndexResponseTests.java | 2 +- .../ingest/SimulateExecutionServiceTests.java | 2 +- .../SimulatePipelineRequestParsingTests.java | 18 +- .../ingest/WriteableIngestDocumentTests.java | 16 +- ...TransportResyncReplicationActionTests.java | 12 +- .../action/search/ExpandSearchPhaseTests.java | 24 +- .../search/SearchPhaseControllerTests.java | 26 +- .../SearchQueryThenFetchAsyncActionTests.java | 16 +- .../action/search/SearchRequestTests.java | 16 + .../search/SearchResponseMergerTests.java | 15 +- .../action/search/SearchResponseTests.java | 6 +- .../search/ShardSearchFailureTests.java | 4 +- .../TransportSearchActionSingleNodeTests.java | 2 +- .../search/TransportSearchActionTests.java | 106 +- .../action/search/TransportSearchIT.java | 6 +- .../action/support/ActiveShardCountTests.java | 24 +- .../support/ActiveShardsObserverIT.java | 6 +- .../action/support/AutoCreateIndexTests.java | 22 +- .../TransportBroadcastByNodeActionTests.java | 45 +- .../nodes/TransportNodesActionTests.java | 3 +- .../ReplicationOperationTests.java | 30 +- ...tReplicationActionRetryOnClosedNodeIT.java | 6 +- .../TransportReplicationActionTests.java | 100 +- ...ReplicationAllPermitsAcquisitionTests.java | 22 +- .../TransportWriteActionTests.java | 18 +- .../TransportMultiTermVectorsActionTests.java | 14 +- .../action/update/UpdateResponseTests.java | 2 +- .../elasticsearch/aliases/IndexAliasesIT.java | 88 +- .../elasticsearch/blocks/SimpleBlocksIT.java | 10 +- .../bootstrap/BootstrapChecksTests.java | 8 +- .../bootstrap/MaxMapCountCheckTests.java | 6 +- .../cluster/ClusterChangedEventTests.java | 180 +- .../cluster/ClusterHealthIT.java | 9 +- .../cluster/ClusterInfoServiceIT.java | 4 +- .../cluster/ClusterModuleTests.java | 25 + .../cluster/ClusterStateDiffIT.java | 150 +- .../cluster/ClusterStateTests.java | 774 +++++++ .../elasticsearch/cluster/DiskUsageTests.java | 18 +- .../cluster/MinimumMasterNodesIT.java | 16 +- .../elasticsearch/cluster/NoMasterNodeIT.java | 6 +- .../cluster/NodeConnectionsServiceTests.java | 4 - .../cluster/SimpleClusterStateIT.java | 76 +- .../cluster/SimpleDataNodesIT.java | 6 +- ...rdFailedClusterStateTaskExecutorTests.java | 30 +- ...dStartedClusterStateTaskExecutorTests.java | 36 +- .../action/shard/ShardStateActionTests.java | 4 +- .../allocation/AwarenessAllocationIT.java | 14 +- .../cluster/allocation/ClusterRerouteIT.java | 28 +- .../allocation/FilteringAllocationIT.java | 14 +- .../allocation/SimpleAllocationIT.java | 2 +- .../ClusterFormationFailureHelperTests.java | 10 +- ...ts.java => CoordinationMetadataTests.java} | 22 +- .../coordination/CoordinationStateTests.java | 115 +- .../coordination/CoordinatorTests.java | 101 +- .../ElasticsearchNodeCommandTests.java | 68 +- .../cluster/coordination/JoinHelperTests.java | 10 +- .../coordination/JoinTaskExecutorTests.java | 40 +- .../cluster/coordination/MessagesTests.java | 4 +- .../cluster/coordination/NodeJoinTests.java | 98 +- .../coordination/PreVoteCollectorTests.java | 9 +- .../coordination/PublicationTests.java | 17 +- .../PublicationTransportHandlerTests.java | 2 +- .../coordination/RareClusterStateIT.java | 34 +- .../coordination/ReconfiguratorTests.java | 2 +- .../coordination/RemoveCustomsCommandIT.java | 6 +- .../coordination/RemoveSettingsCommandIT.java | 6 +- .../UnsafeBootstrapAndDetachCommandIT.java | 18 +- .../coordination/VotingConfigurationIT.java | 4 +- .../cluster/coordination/ZenDiscoveryIT.java | 20 +- .../health/ClusterHealthAllocationTests.java | 16 +- .../health/ClusterIndexHealthTests.java | 20 +- .../health/ClusterStateHealthTests.java | 58 +- ...DataTests.java => AliasMetadataTests.java} | 28 +- .../metadata/AutoExpandReplicasTests.java | 10 +- .../metadata/ComponentTemplateTests.java | 15 +- .../cluster/metadata/DataStreamTests.java | 14 +- .../HumanReadableIndexSettingsTests.java | 14 +- ...xTests.java => IndexAbstractionTests.java} | 44 +- ...DataTests.java => IndexMetadataTests.java} | 139 +- .../IndexNameExpressionResolverTests.java | 360 ++-- ...s.java => IndexTemplateMetadataTests.java} | 36 +- .../metadata/IndexTemplateV2Tests.java | 15 +- .../MetaDataIndexTemplateServiceTests.java | 312 --- ...a => MetadataCreateIndexServiceTests.java} | 241 +-- ...a => MetadataDeleteIndexServiceTests.java} | 18 +- ... => MetadataIndexAliasesServiceTests.java} | 156 +- ...va => MetadataIndexStateServiceTests.java} | 182 +- ...va => MetadataIndexStateServiceUtils.java} | 12 +- .../MetadataIndexTemplateServiceTests.java | 753 +++++++ ... => MetadataIndexUpgradeServiceTests.java} | 108 +- ....java => MetadataMappingServiceTests.java} | 20 +- ...{MetaDataTests.java => MetadataTests.java} | 506 +++-- .../metadata/TemplateUpgradeServiceIT.java | 26 +- .../metadata/TemplateUpgradeServiceTests.java | 40 +- .../metadata/ToAndFromJsonMetaDataTests.java | 344 ---- .../metadata/ToAndFromJsonMetadataTests.java | 626 ++++++ .../metadata/UpgradeIndexSettingsIT.java | 6 +- .../WildcardExpressionResolverTests.java | 74 +- .../cluster/node/DiscoveryNodeTests.java | 51 + .../cluster/routing/AllocationIdIT.java | 8 +- .../cluster/routing/DelayedAllocationIT.java | 22 +- .../DelayedAllocationServiceTests.java | 46 +- .../routing/OperationRoutingTests.java | 46 +- .../cluster/routing/PrimaryAllocationIT.java | 34 +- .../cluster/routing/PrimaryTermsTests.java | 34 +- .../cluster/routing/RoutingNodeTests.java | 26 +- .../routing/RoutingTableGenerator.java | 16 +- .../cluster/routing/RoutingTableTests.java | 82 +- .../cluster/routing/ShardRoutingTests.java | 2 +- .../cluster/routing/UnassignedInfoTests.java | 98 +- .../allocation/AddIncrementallyTests.java | 28 +- .../allocation/AllocationCommandsTests.java | 78 +- .../allocation/AllocationPriorityTests.java | 20 +- .../allocation/AllocationServiceTests.java | 273 +++ .../allocation/AwarenessAllocationTests.java | 108 +- .../allocation/BalanceConfigurationTests.java | 28 +- .../BalanceUnbalancedClusterTests.java | 12 +- .../allocation/CatAllocationTestCase.java | 14 +- .../ClusterRebalanceRoutingTests.java | 123 +- .../ConcurrentRebalanceRoutingTests.java | 12 +- .../allocation/DeadNodesAllocationTests.java | 28 +- .../DecisionsImpactOnClusterHealthTests.java | 12 +- .../allocation/DiskThresholdMonitorTests.java | 60 +- ...ReplicaAsPrimaryDuringRelocationTests.java | 12 +- .../ExpectedShardSizeAllocationTests.java | 24 +- .../allocation/FailedNodeRoutingTests.java | 22 +- .../allocation/FailedShardsRoutingTests.java | 78 +- .../allocation/FilterRoutingTests.java | 46 +- .../allocation/InSyncAllocationIdTests.java | 88 +- .../routing/allocation/IndexBalanceTests.java | 38 +- .../MaxRetryAllocationDeciderTests.java | 20 +- .../NodeVersionAllocationDeciderTests.java | 56 +- ...alPrimariesToRelocatingPrimariesTests.java | 24 +- .../PreferPrimaryAllocationTests.java | 28 +- .../PrimaryElectionRoutingTests.java | 28 +- ...yNotRelocatedWhileBeingRecoveredTests.java | 12 +- .../RandomAllocationDeciderTests.java | 16 +- .../allocation/RebalanceAfterActiveTests.java | 12 +- .../ReplicaAllocatedAfterPrimaryTests.java | 12 +- .../ResizeAllocationDeciderTests.java | 66 +- .../RetryFailedAllocationTests.java | 10 +- .../RoutingNodesIntegrityTests.java | 44 +- .../allocation/SameShardRoutingTests.java | 14 +- .../routing/allocation/ShardStateIT.java | 16 +- .../allocation/ShardVersioningTests.java | 16 +- .../ShardsLimitAllocationTests.java | 62 +- .../SingleShardNoReplicasRoutingTests.java | 42 +- .../SingleShardOneReplicaRoutingTests.java | 12 +- .../allocation/StartedShardsRoutingTests.java | 16 +- .../TenShardsOneReplicaRoutingTests.java | 12 +- .../allocation/ThrottlingAllocationTests.java | 72 +- .../TrackFailedAllocationNodesTests.java | 10 +- .../UpdateNumberOfReplicasTests.java | 24 +- .../decider/AllocationDecidersTests.java | 10 +- .../decider/DiskThresholdDeciderTests.java | 68 +- .../DiskThresholdDeciderUnitTests.java | 84 +- .../EnableAllocationShortCircuitTests.java | 38 +- .../decider/EnableAllocationTests.java | 74 +- .../decider/FilterAllocationDeciderTests.java | 40 +- .../allocation/decider/MockDiskUsagesIT.java | 12 +- ...storeInProgressAllocationDeciderTests.java | 16 +- .../UpdateShardAllocationSettingsIT.java | 14 +- .../ClusterSerializationTests.java | 54 +- .../ClusterStateToStringTests.java | 16 +- .../service/ClusterApplierServiceTests.java | 4 +- .../cluster/settings/ClusterSettingsIT.java | 20 +- .../cluster/shards/ClusterSearchShardsIT.java | 10 +- .../cluster/shards/ClusterShardLimitIT.java | 22 +- .../structure/RoutingIteratorTests.java | 54 +- .../elasticsearch/common/ChannelsTests.java | 4 +- .../elasticsearch/common/RoundingTests.java | 4 +- .../blobstore/fs/FsBlobContainerTests.java | 108 + .../bytes/ReleasableBytesReferenceTests.java | 3 +- .../common/geo/GeoJsonShapeParserTests.java | 10 +- .../common/geo/GeoWKTShapeParserTests.java | 34 +- .../elasticsearch/common/io/StreamsTests.java | 16 + .../common/io/stream/BytesStreamsTests.java | 27 + .../JsonThrowablePatternConverterTests.java | 2 +- .../store/ByteArrayIndexInputTests.java | 47 +- .../rounding/TimeZoneRoundingTests.java | 4 +- .../common/settings/ConsistentSettingsIT.java | 18 +- .../common/settings/ScopedSettingsTests.java | 28 +- .../common/settings/SettingTests.java | 8 +- .../common/settings/UpgradeSettingsIT.java | 16 +- .../common/xcontent/BaseXContentTestCase.java | 6 +- .../xcontent/support/XContentHelperTests.java | 125 ++ .../ClusterDisruptionCleanSettingsIT.java | 6 +- .../discovery/ClusterDisruptionIT.java | 18 +- .../discovery/DiscoveryDisruptionIT.java | 6 +- .../discovery/DiskDisruptionIT.java | 6 +- .../discovery/MasterDisruptionIT.java | 12 +- .../discovery/SnapshotDisruptionIT.java | 99 +- .../single/SingleNodeDiscoveryIT.java | 4 +- .../elasticsearch/document/ShardInfoIT.java | 6 +- .../elasticsearch/env/EnvironmentTests.java | 31 - .../elasticsearch/env/NodeEnvironmentIT.java | 6 +- .../env/NodeEnvironmentTests.java | 42 +- ...aDataTests.java => NodeMetadataTests.java} | 50 +- .../env/NodeRepurposeCommandTests.java | 8 +- .../env/OverrideNodeVersionCommandTests.java | 34 +- .../gateway/ClusterStateUpdatersTests.java | 194 +- .../gateway/DanglingIndicesStateTests.java | 78 +- .../gateway/GatewayIndexStateIT.java | 86 +- .../GatewayMetaStatePersistedStateTests.java | 142 +- .../gateway/GatewayMetaStateTests.java | 168 +- .../IncrementalClusterStateWriterTests.java | 178 +- .../gateway/MetaStateServiceTests.java | 134 +- ...eDataNodesIT.java => MetadataNodesIT.java} | 32 +- ...sts.java => MetadataStateFormatTests.java} | 32 +- .../PersistedClusterStateServiceTests.java | 250 +-- .../gateway/PrimaryShardAllocatorTests.java | 67 +- .../gateway/PriorityComparatorTests.java | 22 +- .../gateway/RecoveryFromGatewayIT.java | 32 +- .../gateway/ReplicaShardAllocatorIT.java | 28 +- .../ReplicaShardAllocatorSyncIdIT.java | 10 +- .../gateway/ReplicaShardAllocatorTests.java | 159 +- .../elasticsearch/index/HiddenIndexIT.java | 8 +- .../elasticsearch/index/IndexModuleTests.java | 24 +- .../index/IndexServiceTests.java | 14 +- .../index/IndexSettingsTests.java | 232 +-- .../index/IndexSortSettingsTests.java | 4 +- .../index/IndexingSlowLogTests.java | 107 +- .../index/MergePolicySettingsTests.java | 26 +- .../index/MergeSchedulerSettingsTests.java | 42 +- .../index/SearchSlowLogTests.java | 108 +- .../index/analysis/AnalysisRegistryTests.java | 71 +- .../index/analysis/PreBuiltAnalyzerTests.java | 4 +- .../PreConfiguredTokenFilterTests.java | 14 +- .../ReloadableCustomAnalyzerTests.java | 4 +- .../index/analysis/StopAnalyzerTests.java | 4 +- .../index/engine/InternalEngineMergeIT.java | 6 +- .../index/engine/InternalEngineTests.java | 95 +- .../index/engine/NoOpEngineRecoveryTests.java | 2 +- .../index/engine/NoOpEngineTests.java | 6 +- .../index/fielddata/FieldDataCacheTests.java | 8 +- .../FieldStatsProviderRefreshTests.java | 4 +- .../index/mapper/DateFieldTypeTests.java | 24 +- .../mapper/DocumentFieldMapperTests.java | 4 +- .../index/mapper/DocumentParserTests.java | 9 +- .../index/mapper/DynamicMappingIT.java | 4 +- .../index/mapper/DynamicMappingTests.java | 10 +- .../mapper/ExternalFieldMapperTests.java | 4 +- .../mapper/FieldFilterMapperPluginTests.java | 56 +- .../mapper/FieldNamesFieldMapperTests.java | 8 +- .../mapper/FieldNamesFieldTypeTests.java | 6 +- .../mapper/GeoShapeFieldMapperTests.java | 3 +- .../index/mapper/IdFieldTypeTests.java | 16 +- .../index/mapper/IndexFieldTypeTests.java | 10 +- .../index/mapper/KeywordFieldMapperTests.java | 12 +- .../LegacyGeoShapeFieldMapperTests.java | 3 +- .../mapper/MapperMergeValidatorTests.java | 4 +- .../index/mapper/MapperServiceTests.java | 6 +- .../index/mapper/MapperTests.java | 4 +- .../mapper/MultiFieldsIntegrationIT.java | 32 +- .../index/mapper/NestedObjectMapperTests.java | 6 +- .../index/mapper/ObjectMapperMergeTests.java | 15 +- .../index/mapper/RangeFieldTypeTests.java | 6 +- .../index/mapper/RootObjectMapperTests.java | 4 +- .../index/mapper/TypeParsersTests.java | 6 +- .../index/mapper/UpdateMappingTests.java | 10 +- .../index/query/BoolQueryBuilderTests.java | 38 +- .../index/query/DisMaxQueryBuilderTests.java | 4 +- .../index/query/FuzzyQueryBuilderTests.java | 24 +- .../query/GeoShapeQueryBuilderTests.java | 8 +- .../query/IntervalQueryBuilderTests.java | 116 +- .../MatchBoolPrefixQueryBuilderTests.java | 22 +- .../MatchPhrasePrefixQueryBuilderTests.java | 2 +- .../query/MatchPhraseQueryBuilderTests.java | 2 +- .../index/query/MatchQueryBuilderTests.java | 66 +- .../query/MoreLikeThisQueryBuilderTests.java | 14 +- .../query/MultiMatchQueryBuilderTests.java | 124 +- .../index/query/NestedQueryBuilderTests.java | 2 +- .../index/query/PrefixQueryBuilderTests.java | 8 +- .../index/query/QueryShardContextTests.java | 22 +- .../query/QueryStringQueryBuilderTests.java | 406 ++-- .../index/query/RandomQueryBuilder.java | 6 +- .../index/query/RangeQueryBuilderTests.java | 2 +- .../index/query/RangeQueryRewriteTests.java | 6 +- .../index/query/RegexpQueryBuilderTests.java | 2 +- .../query/SearchIndexNameMatcherTests.java | 22 +- .../query/SimpleQueryStringBuilderTests.java | 180 +- .../query/SpanTermQueryBuilderTests.java | 6 +- .../index/query/TermQueryBuilderTests.java | 2 +- .../index/query/TermsQueryBuilderTests.java | 8 +- .../query/TermsSetQueryBuilderTests.java | 13 +- .../query/WildcardQueryBuilderTests.java | 8 +- .../index/query/WrapperQueryBuilderTests.java | 12 +- .../FunctionScoreQueryBuilderTests.java | 23 +- .../ScoreFunctionBuilderTests.java | 8 +- .../WorkerBulkByScrollTaskStateTests.java | 5 +- .../IndexLevelReplicationTests.java | 20 +- .../RecoveryDuringReplicationTests.java | 20 +- .../RetentionLeasesReplicationTests.java | 12 +- .../PeerRecoveryRetentionLeaseCreationIT.java | 12 +- .../index/seqno/RetentionLeasesTests.java | 2 +- .../index/shard/IndexShardIT.java | 26 +- .../index/shard/IndexShardTests.java | 292 +-- .../index/shard/NewPathForShardTests.java | 6 +- .../RemoveCorruptedShardDataCommandIT.java | 18 +- .../RemoveCorruptedShardDataCommandTests.java | 28 +- .../index/shard/ShardGetServiceTests.java | 32 +- .../index/shard/ShardIdTests.java | 6 +- .../index/shard/ShardPathTests.java | 16 +- .../index/shard/ShardSplittingQueryTests.java | 40 +- .../index/shard/StoreRecoveryTests.java | 12 +- .../snapshots/blobstore/FileInfoTests.java | 14 +- .../index/store/CorruptedFileIT.java | 32 +- .../index/store/FsDirectoryFactoryTests.java | 4 +- .../elasticsearch/index/store/StoreTests.java | 78 +- .../index/suggest/stats/SuggestStatsIT.java | 4 +- .../index/translog/TranslogTests.java | 8 +- .../elasticsearch/indexing/IndexActionIT.java | 10 +- .../IndexLifecycleActionIT.java | 4 +- ...DateMathIndexExpressionsIntegrationIT.java | 20 +- .../indices/IndexingMemoryControllerIT.java | 2 +- .../IndexingMemoryControllerTests.java | 6 +- .../indices/IndicesLifecycleListenerIT.java | 10 +- ...dicesLifecycleListenerSingleNodeTests.java | 10 +- .../indices/IndicesModuleTests.java | 6 +- .../indices/IndicesRequestCacheIT.java | 22 +- .../indices/IndicesServiceCloseTests.java | 4 +- .../indices/IndicesServiceTests.java | 112 +- .../indices/analysis/AnalysisModuleTests.java | 18 +- ...actIndicesClusterStateServiceTestCase.java | 30 +- .../indices/cluster/ClusterStateChanges.java | 70 +- ...ClusterStateServiceRandomUpdatesTests.java | 58 +- .../elasticsearch/indices/flush/FlushIT.java | 4 +- .../mapping/SimpleGetFieldMappingsIT.java | 16 +- .../indices/mapping/SimpleGetMappingsIT.java | 14 +- .../mapping/UpdateMappingIntegrationIT.java | 14 +- .../memory/breaker/CircuitBreakerNoopIT.java | 2 +- .../breaker/CircuitBreakerServiceIT.java | 10 +- .../indices/recovery/IndexRecoveryIT.java | 86 +- .../PeerRecoveryTargetServiceTests.java | 26 +- .../recovery/RecoverySourceHandlerTests.java | 70 +- .../indices/recovery/RecoveryStatusTests.java | 6 +- .../indices/recovery/RecoveryTests.java | 22 +- .../recovery/ReplicaToPrimaryPromotionIT.java | 10 +- .../indices/settings/GetSettingsBlocksIT.java | 10 +- .../InternalOrPrivateSettingsPlugin.java | 12 +- .../settings/UpdateNumberOfReplicasIT.java | 34 +- .../indices/settings/UpdateSettingsIT.java | 92 +- .../state/CloseIndexDisableCloseAllIT.java | 8 +- .../indices/state/CloseIndexIT.java | 58 +- .../state/CloseWhileRelocatingShardsIT.java | 2 +- .../indices/state/OpenCloseIndexIT.java | 22 +- .../indices/state/ReopenWhileClosingIT.java | 6 +- .../indices/state/SimpleIndexStateIT.java | 10 +- .../indices/stats/IndexStatsIT.java | 12 +- .../store/IndicesStoreIntegrationIT.java | 34 +- .../template/SimpleIndexTemplateIT.java | 34 +- .../ingest/IngestDocumentTests.java | 8 +- .../ingest/IngestMetadataTests.java | 4 +- .../ingest/IngestServiceTests.java | 24 +- .../org/elasticsearch/mget/SimpleMgetIT.java | 4 +- .../jvm/JvmGcMonitorServiceSettingsTests.java | 20 +- .../org/elasticsearch/node/NodeTests.java | 4 +- .../DestructiveOperationsIT.java | 10 +- .../PersistentTasksClusterServiceTests.java | 184 +- ...> PersistentTasksCustomMetadataTests.java} | 86 +- .../PersistentTasksDecidersTestCase.java | 18 +- .../PersistentTasksExecutorFullRestartIT.java | 12 +- .../persistent/PersistentTasksExecutorIT.java | 16 +- .../PersistentTasksExecutorResponseTests.java | 6 +- .../PersistentTasksNodeServiceTests.java | 50 +- .../persistent/TestPersistentTasksPlugin.java | 4 +- .../decider/EnableAssignmentDeciderIT.java | 10 +- .../recovery/FullRollingRestartIT.java | 6 +- .../recovery/RecoveryWhileUnderLoadIT.java | 4 +- .../elasticsearch/recovery/RelocationIT.java | 12 +- .../recovery/TruncatedRecoveryIT.java | 8 +- .../repositories/RepositoriesServiceIT.java | 10 +- .../RepositoriesServiceTests.java | 25 +- .../BlobStoreRepositoryRestoreTests.java | 33 +- .../blobstore/BlobStoreRepositoryTests.java | 8 +- .../repositories/fs/FsRepositoryTests.java | 10 +- .../rest/action/RestActionsTests.java | 4 +- .../RestClusterGetSettingsActionTests.java | 12 +- .../cluster/RestNodesInfoActionTests.java | 85 +- .../indices/RestGetAliasesActionTests.java | 34 +- .../cat/RestCatRecoveryActionTests.java | 8 - .../action/cat/RestIndicesActionTests.java | 37 +- .../action/document/RestIndexActionTests.java | 7 +- .../routing/AliasResolveRoutingIT.java | 22 +- .../elasticsearch/routing/AliasRoutingIT.java | 4 +- .../script/ScriptCacheTests.java | 14 +- ...ataTests.java => ScriptMetadataTests.java} | 66 +- .../script/ScriptServiceTests.java | 87 +- .../script/StoredScriptTests.java | 6 +- .../search/DefaultSearchContextTests.java | 12 +- .../elasticsearch/search/SearchHitTests.java | 30 +- .../elasticsearch/search/SearchHitsTests.java | 10 +- .../search/SearchModuleTests.java | 35 +- .../search/SearchServiceTests.java | 131 +- .../aggregations/FiltersAggsRewriteIT.java | 4 +- .../InternalAggregationsTests.java | 74 +- .../InternalMultiBucketAggregationTests.java | 31 +- .../{MetaDataIT.java => MetadataIT.java} | 36 +- .../aggregations/TestAggregatorFactory.java | 9 +- .../bucket/DiversifiedSamplerIT.java | 4 +- .../aggregations/bucket/FiltersTests.java | 4 +- .../aggregations/bucket/GeoDistanceIT.java | 6 +- .../aggregations/bucket/GeoHashGridIT.java | 4 +- .../aggregations/bucket/MissingTests.java | 2 +- .../search/aggregations/bucket/NestedIT.java | 4 +- .../aggregations/bucket/RareTermsTests.java | 2 +- .../aggregations/bucket/ReverseNestedIT.java | 4 +- .../search/aggregations/bucket/SamplerIT.java | 4 +- .../SignificantTermsSignificanceScoreIT.java | 4 +- .../bucket/SignificantTermsTests.java | 2 +- .../bucket/TermsDocCountErrorIT.java | 10 +- .../bucket/TermsShardMinDocCountIT.java | 4 +- .../aggregations/bucket/TermsTests.java | 2 +- ...djacencyMatrixAggregationBuilderTests.java | 16 +- .../InternalAdjacencyMatrixTests.java | 21 +- .../composite/CompositeAggregatorTests.java | 4 +- .../composite/InternalCompositeTests.java | 25 +- .../bucket/filter/InternalFilterTests.java | 5 +- .../bucket/filter/InternalFiltersTests.java | 21 +- .../bucket/geogrid/GeoGridTestCase.java | 25 +- .../bucket/geogrid/GeoHashGridTests.java | 5 +- .../bucket/geogrid/GeoTileGridTests.java | 5 +- .../bucket/global/InternalGlobalTests.java | 5 +- .../AutoDateHistogramAggregatorTests.java | 4 +- .../bucket/histogram/DateHistogramTests.java | 12 + .../bucket/histogram/ExtendedBoundsTests.java | 10 +- .../InternalAutoDateHistogramTests.java | 119 +- .../histogram/InternalDateHistogramTests.java | 21 +- .../histogram/InternalHistogramTests.java | 23 +- .../NumericHistogramAggregatorTests.java | 49 + .../bucket/missing/InternalMissingTests.java | 5 +- .../missing/MissingAggregatorTests.java | 259 +-- .../bucket/nested/InternalNestedTests.java | 5 +- .../nested/InternalReverseNestedTests.java | 5 +- .../bucket/nested/NestedAggregatorTests.java | 14 +- .../range/DateRangeAggregatorTests.java | 4 +- .../range/InternalBinaryRangeTests.java | 19 +- .../bucket/range/InternalDateRangeTests.java | 19 +- .../range/InternalGeoDistanceTests.java | 19 +- .../bucket/range/InternalRangeTestCase.java | 17 +- .../bucket/range/InternalRangeTests.java | 19 +- .../bucket/range/RangeAggregatorTests.java | 2 +- .../sampler/DiversifiedSamplerTests.java | 4 +- .../bucket/sampler/InternalSamplerTests.java | 5 +- .../InternalSignificantTermsTestCase.java | 18 +- .../SignificanceHeuristicTests.java | 13 +- .../SignificantLongTermsTests.java | 34 +- .../SignificantStringTermsTests.java | 34 +- .../SignificantTermsAggregatorTests.java | 16 +- .../terms/BinaryTermsAggregatorTests.java | 176 ++ .../bucket/terms/DoubleTermsTests.java | 34 +- .../bucket/terms/InternalTermsTestCase.java | 21 +- .../terms/KeywordTermsAggregatorTests.java | 161 ++ .../bucket/terms/LongTermsTests.java | 34 +- .../terms/NumericTermsAggregatorTests.java | 190 ++ .../terms/RareTermsAggregatorTests.java | 33 +- .../bucket/terms/StringTermsTests.java | 34 +- .../bucket/terms/TermsAggregatorTests.java | 106 +- .../metrics/AbstractPercentilesTestCase.java | 11 +- .../metrics/AvgAggregatorTests.java | 2 +- .../metrics/CardinalityAggregatorTests.java | 10 +- .../metrics/CardinalityTests.java | 3 +- .../metrics/ExtendedStatsAggregatorTests.java | 2 + .../metrics/GeoCentroidAggregatorTests.java | 13 + .../HDRPercentileRanksAggregatorTests.java | 4 +- .../HDRPercentilesAggregatorTests.java | 57 +- .../metrics/InternalAvgTests.java | 22 +- .../metrics/InternalCardinalityTests.java | 22 +- .../metrics/InternalExtendedStatsTests.java | 29 +- .../metrics/InternalGeoBoundsTests.java | 21 +- .../metrics/InternalGeoCentroidTests.java | 21 +- .../InternalHDRPercentilesRanksTests.java | 22 +- .../metrics/InternalHDRPercentilesTests.java | 27 +- .../metrics/InternalMaxTests.java | 18 +- .../InternalMedianAbsoluteDeviationTests.java | 20 +- .../metrics/InternalMinTests.java | 18 +- .../metrics/InternalScriptedMetricTests.java | 24 +- .../metrics/InternalStatsBucketTests.java | 9 +- .../metrics/InternalStatsTests.java | 31 +- .../metrics/InternalSumTests.java | 22 +- .../InternalTDigestPercentilesRanksTests.java | 24 +- .../InternalTDigestPercentilesTests.java | 24 +- .../metrics/InternalTopHitsTests.java | 23 +- .../metrics/InternalValueCountTests.java | 21 +- .../metrics/InternalWeightedAvgTests.java | 29 +- .../metrics/MaxAggregatorTests.java | 9 +- .../metrics/MedianAbsoluteDeviationIT.java | 4 +- .../metrics/MinAggregatorTests.java | 69 +- .../ScriptedMetricAggregatorTests.java | 5 +- .../metrics/StatsAggregatorTests.java | 2 + ...TDigestPercentileRanksAggregatorTests.java | 4 +- .../TDigestPercentilesAggregatorTests.java | 4 +- .../metrics/ValueCountAggregatorTests.java | 35 +- .../aggregations/metrics/ValueCountTests.java | 2 +- .../aggregations/pipeline/AvgBucketTests.java | 2 +- .../pipeline/BucketHelpersTests.java | 4 +- .../pipeline/BucketScriptAggregatorTests.java | 2 +- ...ScriptPipelineAggregationBuilderTests.java | 48 + .../pipeline/BucketSelectorTests.java | 8 + .../pipeline/BucketSortTests.java | 10 +- .../pipeline/ExtendedStatsBucketTests.java | 2 +- .../InternalBucketMetricValueTests.java | 26 +- .../pipeline/InternalDerivativeTests.java | 25 +- .../InternalExtendedStatsBucketTests.java | 15 +- .../InternalPercentilesBucketTests.java | 41 +- .../pipeline/InternalSimpleValueTests.java | 22 +- .../aggregations/pipeline/MaxBucketTests.java | 2 +- .../aggregations/pipeline/MinBucketTests.java | 2 +- .../pipeline/PercentilesBucketTests.java | 2 +- .../PipelineAggregationHelperTests.java | 2 +- .../aggregations/pipeline/SerialDiffIT.java | 7 +- .../pipeline/StatsBucketTests.java | 2 +- .../aggregations/pipeline/SumBucketTests.java | 2 +- .../support/CoreValuesSourceTypeTests.java | 43 +- .../aggregations/support/ValueTypeTests.java | 55 +- .../support/ValuesSourceConfigTests.java | 89 +- .../basic/TransportTwoNodesSearchIT.java | 2 +- .../search/fetch/FetchSubPhasePluginIT.java | 2 +- .../fetch/subphase/FetchSourcePhaseTests.java | 2 +- .../highlight/HighlightBuilderTests.java | 6 +- .../search/fields/SearchFieldsIT.java | 2 +- .../functionscore/DecayFunctionScoreIT.java | 6 +- .../search/functionscore/QueryRescorerIT.java | 4 +- .../search/geo/GeoBoundingBoxQueryIT.java | 8 +- .../search/geo/GeoDistanceIT.java | 4 +- .../elasticsearch/search/geo/GeoFilterIT.java | 4 +- .../search/geo/GeoPolygonIT.java | 4 +- .../search/geo/GeoQueryTests.java | 9 + .../internal/ShardSearchRequestTests.java | 116 +- .../search/morelikethis/MoreLikeThisIT.java | 4 +- .../search/preference/SearchPreferenceIT.java | 6 +- .../search/query/MultiMatchQueryIT.java | 4 +- .../search/query/QuerySearchResultTests.java | 14 - .../search/query/SearchQueryIT.java | 2 +- .../rescore/QueryRescorerBuilderTests.java | 10 +- .../search/scroll/DuelScrollIT.java | 6 +- .../search/scroll/SearchScrollIT.java | 16 +- .../search/searchafter/SearchAfterIT.java | 2 +- .../search/simple/SimpleSearchIT.java | 4 +- .../search/slice/SliceBuilderTests.java | 14 +- .../search/sort/AbstractSortTestCase.java | 6 +- .../search/sort/FieldSortIT.java | 8 +- .../search/sort/GeoDistanceIT.java | 10 +- .../search/sort/GeoDistanceSortBuilderIT.java | 8 +- .../search/stats/SearchStatsIT.java | 4 +- .../AbstractSuggestionBuilderTestCase.java | 6 +- .../suggest/CompletionSuggestSearchIT.java | 4 +- .../search/suggest/SuggestSearchIT.java | 4 +- .../AbstractSnapshotIntegTestCase.java | 6 +- .../snapshots/BlobStoreFormatIT.java | 4 +- .../snapshots/BlobStoreIncrementalityIT.java | 9 +- .../CorruptedBlobStoreRepositoryIT.java | 10 +- .../DedicatedClusterSnapshotRestoreIT.java | 106 +- ...etadataLoadingDuringSnapshotRestoreIT.java | 16 +- .../snapshots/RepositoriesIT.java | 40 +- ...positoriesMetadataSerializationTests.java} | 46 +- .../RepositoryFilterUserMetadataIT.java | 23 +- .../SharedClusterSnapshotRestoreIT.java | 92 +- .../snapshots/SnapshotResiliencyTests.java | 136 +- .../MockEventuallyConsistentRepository.java | 18 +- ...ckEventuallyConsistentRepositoryTests.java | 50 +- .../threadpool/ThreadPoolTests.java | 33 + .../transport/InboundAggregatorTests.java | 266 +++ .../transport/InboundDecoderTests.java | 310 +++ .../transport/InboundHandlerTests.java | 86 +- .../transport/InboundMessageTests.java | 233 --- .../transport/InboundPipelineTests.java | 303 +++ .../transport/OutboundHandlerTests.java | 225 +- .../transport/RemoteClusterClientTests.java | 9 +- .../transport/RemoteClusterServiceTests.java | 9 +- .../transport/RemoteClusterSettingsTests.java | 16 +- .../transport/TcpTransportTests.java | 50 +- .../transport/TransportDecompressorTests.java | 128 ++ .../transport/TransportHandshakerTests.java | 88 +- .../usage/UsageServiceTests.java | 53 + .../validate/SimpleValidateQueryIT.java | 2 +- .../ConcurrentSeqNoVersioningIT.java | 6 +- .../versioning/SimpleVersioningIT.java | 6 +- settings.gradle | 2 +- .../fixtures/azure-fixture/docker-compose.yml | 9 + test/fixtures/hdfs-fixture/build.gradle | 2 +- test/fixtures/s3-fixture/docker-compose.yml | 15 + .../main/java/fixture/s3/S3HttpHandler.java | 10 +- .../ClusterStateCreationUtils.java | 70 +- .../cluster/ESAllocationTestCase.java | 32 +- .../MockInternalClusterInfoService.java | 2 +- .../AbstractCoordinatorTestCase.java | 57 +- .../CoordinationStateTestCluster.java | 44 +- .../coordination/DeterministicTaskQueue.java | 5 - .../cluster/routing/TestShardRouting.java | 10 +- .../common/breaker/TestCircuitBreaker.java | 47 + .../common/logging/JsonLogLine.java | 51 +- .../common/logging/JsonLogsIntegTestCase.java | 33 +- .../common/logging/JsonLogsStream.java | 15 +- .../lucene/store/ESIndexInputTestCase.java | 192 ++ .../gateway/MockGatewayMetaState.java | 17 +- .../elasticsearch/index/MapperTestUtils.java | 6 +- .../index/RandomCreateIndexGenerator.java | 8 +- .../index/analysis/AnalysisTestsHelper.java | 6 +- .../index/engine/EngineTestCase.java | 12 +- .../index/mapper/MockFieldMapper.java | 4 +- .../ESIndexLevelReplicationTestCase.java | 46 +- .../index/shard/IndexShardTestCase.java | 58 +- .../index/shard/RestoreOnlyRepository.java | 17 +- .../indices/recovery/AsyncRecoveryTarget.java | 10 +- .../AbstractThirdPartyRepositoryTestCase.java | 8 +- .../blobstore/BlobStoreTestUtil.java | 36 +- .../ESBlobStoreRepositoryIntegTestCase.java | 12 +- ...ESMockAPIBasedRepositoryIntegTestCase.java | 6 +- .../aggregations/AggregatorTestCase.java | 65 +- .../aggregations/BaseAggregationTestCase.java | 4 +- .../BasePipelineAggregationTestCase.java | 4 +- ...ternalSingleBucketAggregationTestCase.java | 30 +- .../mockstore/BlobContainerWrapper.java | 16 +- .../snapshots/mockstore/MockRepository.java | 20 +- .../test/AbstractBootstrapCheckTestCase.java | 8 +- .../test/AbstractBuilderTestCase.java | 28 +- ...AbstractDiffableSerializationTestCase.java | 2 +- .../test/AbstractQueryTestCase.java | 8 +- .../elasticsearch/test/ESIntegTestCase.java | 133 +- .../test/ESSingleNodeTestCase.java | 20 +- .../org/elasticsearch/test/ESTestCase.java | 6 +- .../test/ESTokenStreamTestCase.java | 4 +- .../test/IndexSettingsModule.java | 28 +- .../test/InternalAggregationTestCase.java | 144 +- ...nternalMultiBucketAggregationTestCase.java | 13 +- .../test/InternalSettingsPlugin.java | 4 +- .../test/InternalTestCluster.java | 6 +- .../org/elasticsearch/test/RandomObjects.java | 2 +- .../org/elasticsearch/test/TestCluster.java | 10 +- ...mMetaData.java => TestCustomMetadata.java} | 16 +- .../test/TestGeoShapeFieldMapperPlugin.java | 46 + .../test/gateway/TestGatewayAllocator.java | 21 +- .../hamcrest/ElasticsearchAssertions.java | 14 +- .../test/rest/ESRestTestCase.java | 60 +- .../test/store/MockFSDirectoryFactory.java | 6 +- .../test/transport/MockTransport.java | 4 - .../test/transport/StubbableTransport.java | 5 - .../AbstractSimpleTransportTestCase.java | 24 +- .../elasticsearch/transport/TestRequest.java | 24 +- .../elasticsearch/transport/TestResponse.java | 43 + .../transport/TestTransportChannel.java | 51 + .../transport/nio/MockNioTransport.java | 41 +- .../FakeThreadPoolMasterServiceTests.java | 20 +- .../search/MockSearchServiceTests.java | 8 +- .../ElasticsearchAssertionsTests.java | 10 +- .../security/oidc-authenticate-api.asciidoc | 2 +- .../authentication/oidc-guide.asciidoc | 39 +- .../authentication/saml-guide.asciidoc | 39 +- .../docs/en/security/troubleshooting.asciidoc | 16 +- .../smoketest/XDocsClientYamlTestSuiteIT.java | 4 +- x-pack/plugin/analytics/build.gradle | 2 + .../licenses/commons-math3-3.2.jar.sha1 | 1 + .../licenses/commons-math3-LICENSE.txt | 475 +++++ .../licenses/commons-math3-NOTICE.txt | 9 + .../xpack/analytics/AnalyticsPlugin.java | 35 +- .../xpack/analytics/AnalyticsUsage.java | 6 +- ...ctHistoBackedHDRPercentilesAggregator.java | 119 ++ ...stoBackedTDigestPercentilesAggregator.java | 111 + ...AnalyticsPercentilesAggregatorFactory.java | 61 + ...stoBackedHDRPercentileRanksAggregator.java | 55 + .../HistoBackedHDRPercentilesAggregator.java | 55 + ...ackedTDigestPercentileRanksAggregator.java | 58 + ...stoBackedTDigestPercentilesAggregator.java | 58 + .../support/AnalyticsValuesSourceType.java | 59 + .../support/HistogramValuesSource.java | 51 + .../boxplot/BoxplotAggregationBuilder.java | 31 +- .../analytics/boxplot/BoxplotAggregator.java | 16 +- .../boxplot/BoxplotAggregatorFactory.java | 41 +- .../boxplot/BoxplotAggregatorSupplier.java | 28 + .../analytics/boxplot/InternalBoxplot.java | 8 +- ...CardinalityPipelineAggregationBuilder.java | 4 +- ...mulativeCardinalityPipelineAggregator.java | 2 +- .../InternalSimpleLongValue.java | 6 +- .../mapper/HistogramFieldMapper.java | 9 +- .../stringstats/InternalStringStats.java | 8 +- .../StringStatsAggregationBuilder.java | 40 +- .../stringstats/StringStatsAggregator.java | 12 +- .../StringStatsAggregatorFactory.java | 54 +- .../StringStatsAggregatorSupplier.java | 26 + .../topmetrics/InternalTopMetrics.java | 16 +- .../TopMetricsAggregationBuilder.java | 15 +- .../topmetrics/TopMetricsAggregator.java | 10 +- .../TopMetricsAggregatorFactory.java | 18 +- .../xpack/analytics/ttest/InternalTTest.java | 99 + .../ttest/PairedTTestAggregator.java | 88 + .../analytics/ttest/PairedTTestState.java | 91 + .../xpack/analytics/ttest/TTest.java | 15 + .../ttest/TTestAggregationBuilder.java | 140 ++ .../analytics/ttest/TTestAggregator.java | 64 + .../ttest/TTestAggregatorFactory.java | 75 + .../xpack/analytics/ttest/TTestState.java | 20 + .../xpack/analytics/ttest/TTestStats.java | 85 + .../analytics/ttest/TTestStatsBuilder.java | 78 + .../xpack/analytics/ttest/TTestType.java | 25 + .../ttest/UnpairedTTestAggregator.java | 97 + .../analytics/ttest/UnpairedTTestState.java | 120 ++ .../TransportAnalyticsStatsActionTests.java | 31 +- .../boxplot/BoxplotAggregatorTests.java | 15 +- .../boxplot/InternalBoxplotTests.java | 20 +- .../CumulativeCardinalityAggregatorTests.java | 5 +- ...regatedPercentileRanksAggregatorTests.java | 31 +- ...eAggregatedPercentilesAggregatorTests.java | 33 +- ...regatedPercentileRanksAggregatorTests.java | 29 + ...eAggregatedPercentilesAggregatorTests.java | 33 +- .../stringstats/InternalStringStatsTests.java | 25 +- .../StringStatsAggregatorTests.java | 198 +- .../InternalTopMetricsReduceTests.java | 5 +- .../topmetrics/InternalTopMetricsTests.java | 33 +- .../topmetrics/TopMetricsAggregatorTests.java | 7 +- .../analytics/ttest/InternalTTestTests.java | 140 ++ .../ttest/TTestAggregationBuilderTests.java | 84 + .../analytics/ttest/TTestAggregatorTests.java | 558 +++++ x-pack/plugin/async-search/build.gradle | 21 +- .../plugin/async-search/qa/rest/build.gradle | 27 + .../query/DeprecatedQueryBuilder.java | 75 + .../query/DeprecatedQueryPlugin.java | 24 + .../elasticsearch/qa/AsyncSearchRestIT.java | 23 + .../test/async-search/10_deprecation.yml | 104 + .../async-search/qa/security/build.gradle | 1 + .../xpack/search/AsyncSearchSecurityIT.java | 6 +- .../xpack/search/AsyncSearchIndexService.java | 34 +- .../xpack/search/AsyncSearchTask.java | 37 +- .../xpack/search/MutableSearchResponse.java | 32 +- .../search/RestGetAsyncSearchAction.java | 4 +- .../search/RestSubmitAsyncSearchAction.java | 8 +- .../TransportDeleteAsyncSearchAction.java | 3 +- .../search/TransportGetAsyncSearchAction.java | 3 +- .../TransportSubmitAsyncSearchAction.java | 40 +- ...ionTests.java => AsyncSearchActionIT.java} | 9 +- .../search/AsyncSearchIndexServiceTests.java | 6 +- .../search/AsyncSearchIntegTestCase.java | 2 +- .../search/CancellingAggregationBuilder.java | 14 +- .../RestSubmitAsyncSearchActionTests.java | 96 + .../search/SubmitAsyncSearchRequestTests.java | 4 +- x-pack/plugin/autoscaling/build.gradle | 21 + .../autoscaling/delete_autoscaling_policy.yml | 33 + .../autoscaling/get_autoscaling_decision.yml | 2 + .../autoscaling/get_autoscaling_policy.yml | 34 + .../autoscaling/put_autoscaling_policy.yml | 23 + .../xpack/autoscaling/Autoscaling.java | 69 +- .../autoscaling/AutoscalingMetadata.java | 169 ++ .../action/DeleteAutoscalingPolicyAction.java | 71 + .../action/GetAutoscalingDecisionAction.java | 31 +- .../action/GetAutoscalingPolicyAction.java | 120 ++ .../action/PutAutoscalingPolicyAction.java | 92 + ...ransportDeleteAutoscalingPolicyAction.java | 118 ++ .../TransportGetAutoscalingPolicyAction.java | 90 + .../TransportPutAutoscalingPolicyAction.java | 121 ++ .../decision/AlwaysAutoscalingDecider.java | 74 + .../decision/AutoscalingDecider.java | 31 + .../{ => decision}/AutoscalingDecision.java | 2 +- .../AutoscalingDecisionType.java | 2 +- .../{ => decision}/AutoscalingDecisions.java | 8 +- .../autoscaling/policy/AutoscalingPolicy.java | 120 ++ .../policy/AutoscalingPolicyMetadata.java | 85 + .../RestDeleteAutoscalingPolicyHandler.java | 38 + .../rest/RestGetAutoscalingPolicyHandler.java | 38 + .../rest/RestPutAutoscalingPolicyHandler.java | 43 + .../autoscaling/AutoscalingIntegTestCase.java | 30 + ...ingMetadataDiffableSerializationTests.java | 81 + .../autoscaling/AutoscalingTestCase.java | 82 +- .../autoscaling/LocalStateAutoscaling.java | 27 + ...licyActionRequestWireSerializingTests.java | 25 + ...sionActionRequestWireSerializingTests.java | 25 + ...ionActionResponseWireSerializingTests.java | 37 + ...licyActionRequestWireSerializingTests.java | 25 + ...icyActionResponseWireSerializingTests.java | 34 + ...licyActionRequestWireSerializingTests.java | 34 + ...nsportDeleteAutoscalingPolicyActionIT.java | 55 + ...ortDeleteAutoscalingPolicyActionTests.java | 126 ++ ...TransportGetAutoscalingPolicyActionIT.java | 41 + ...nsportGetAutoscalingPolicyActionTests.java | 111 + ...TransportPutAutoscalingPolicyActionIT.java | 66 + ...nsportPutAutoscalingPolicyActionTests.java | 164 ++ .../AutoscalingDecisionTests.java | 3 +- ...alingDecisionTypeWireSerializingTests.java | 2 +- ...toscalingDecisionWireSerializingTests.java | 3 +- .../AutoscalingDecisionsTests.java | 4 +- ...oscalingDecisionsWireSerializingTests.java | 3 +- ...icyMetadataDiffableSerializationTests.java | 65 + .../AutoscalingPolicySerializingTests.java | 53 + .../xpack/ccr/FollowIndexIT.java | 11 +- .../java/org/elasticsearch/xpack/ccr/Ccr.java | 4 +- .../xpack/ccr/CcrLicenseChecker.java | 26 +- .../ccr/action/AutoFollowCoordinator.java | 54 +- .../xpack/ccr/action/CcrRequests.java | 40 +- .../xpack/ccr/action/ShardChangesAction.java | 24 +- .../ccr/action/ShardFollowTaskCleaner.java | 14 +- .../ccr/action/ShardFollowTasksExecutor.java | 100 +- ...nsportActivateAutoFollowPatternAction.java | 6 +- ...ransportDeleteAutoFollowPatternAction.java | 6 +- .../ccr/action/TransportFollowInfoAction.java | 12 +- .../action/TransportFollowStatsAction.java | 14 +- .../action/TransportForgetFollowerAction.java | 2 +- .../TransportGetAutoFollowPatternAction.java | 8 +- .../action/TransportPauseFollowAction.java | 14 +- .../TransportPutAutoFollowPatternAction.java | 26 +- .../ccr/action/TransportPutFollowAction.java | 12 +- .../action/TransportResumeFollowAction.java | 62 +- .../ccr/action/TransportUnfollowAction.java | 48 +- .../PutCcrRestoreSessionAction.java | 20 +- .../xpack/ccr/repository/CcrRepository.java | 105 +- .../repository/CcrRestoreSourceService.java | 6 +- .../elasticsearch/xpack/CcrIntegTestCase.java | 22 +- .../xpack/CcrSingleNodeTestCase.java | 4 +- .../elasticsearch/xpack/ccr/AutoFollowIT.java | 74 +- .../ccr/CCRInfoTransportActionTests.java | 18 +- .../elasticsearch/xpack/ccr/CcrAliasesIT.java | 46 +- .../elasticsearch/xpack/ccr/CcrLicenseIT.java | 4 +- .../xpack/ccr/CcrRepositoryIT.java | 30 +- .../xpack/ccr/CcrRetentionLeaseIT.java | 48 +- .../org/elasticsearch/xpack/ccr/CcrTests.java | 10 +- .../xpack/ccr/CloseFollowerIndexIT.java | 12 +- .../xpack/ccr/FollowStatsIT.java | 8 +- .../xpack/ccr/FollowerFailOverIT.java | 10 +- .../xpack/ccr/IndexFollowingIT.java | 122 +- .../xpack/ccr/LocalIndexFollowingIT.java | 6 +- .../action/AutoFollowCoordinatorTests.java | 196 +- .../ccr/action/ShardChangesActionTests.java | 2 +- .../ShardFollowNodeTaskStatusTests.java | 2 +- .../ShardFollowTaskReplicationTests.java | 14 +- ...ardFollowTasksExecutorAssignmentTests.java | 105 + .../xpack/ccr/action/StatsResponsesTests.java | 2 +- ...tActivateAutoFollowPatternActionTests.java | 10 +- ...ortDeleteAutoFollowPatternActionTests.java | 10 +- .../TransportFollowInfoActionTests.java | 18 +- .../TransportFollowStatsActionTests.java | 24 +- ...nsportGetAutoFollowPatternActionTests.java | 22 +- ...nsportPutAutoFollowPatternActionTests.java | 32 +- .../TransportResumeFollowActionTests.java | 102 +- .../action/TransportUnfollowActionTests.java | 42 +- .../index/engine/FollowingEngineTests.java | 40 +- .../CcrRepositoryRetentionLeaseTests.java | 14 +- .../CcrRestoreSourceServiceTests.java | 18 +- .../collector/ccr/StatsCollectorTests.java | 4 +- .../elasticsearch/license/LicenseService.java | 98 +- ...sesMetaData.java => LicensesMetadata.java} | 38 +- .../org/elasticsearch/license/Licensing.java | 10 +- .../license/RemoteClusterLicenseChecker.java | 2 +- .../license/StartBasicClusterTask.java | 22 +- .../license/StartTrialClusterTask.java | 20 +- .../StartupSelfGeneratedLicenseTask.java | 60 +- .../TransportGetBasicStatusAction.java | 6 +- .../TransportGetTrialStatusAction.java | 4 +- .../license/XPackLicenseState.java | 73 +- .../SourceOnlySnapshotRepository.java | 36 +- .../xpack/ccr/CCRUsageTransportAction.java | 10 +- .../xpack/core/XPackClientPlugin.java | 40 +- .../elasticsearch/xpack/core/XPackPlugin.java | 18 +- .../action/AnalyticsStatsAction.java | 22 +- .../xpack/core/ccr/AutoFollowMetadata.java | 8 +- .../validation/SourceDestValidator.java | 4 +- .../deprecation/DeprecationInfoAction.java | 8 +- .../NodesDeprecationCheckAction.java | 4 +- .../core/ilm/AbstractUnfollowIndexStep.java | 8 +- .../xpack/core/ilm/AllocateAction.java | 10 +- .../xpack/core/ilm/AllocationRoutedStep.java | 4 +- .../core/ilm/AsyncActionBranchingStep.java | 105 + .../xpack/core/ilm/AsyncActionStep.java | 6 +- .../AsyncRetryDuringSnapshotActionStep.java | 16 +- .../xpack/core/ilm/AsyncWaitStep.java | 4 +- .../xpack/core/ilm/BranchingStep.java | 6 +- .../xpack/core/ilm/CheckShrinkReadyStep.java | 6 +- .../xpack/core/ilm/CleanupSnapshotStep.java | 82 + .../core/ilm/CloseFollowerIndexStep.java | 10 +- .../xpack/core/ilm/CloseIndexStep.java | 8 +- .../core/ilm/CopyExecutionStateStep.java | 76 +- .../xpack/core/ilm/CopySettingsStep.java | 112 + .../xpack/core/ilm/CreateSnapshotStep.java | 83 + .../xpack/core/ilm/DeleteAction.java | 51 +- .../xpack/core/ilm/DeleteStep.java | 6 +- .../xpack/core/ilm/ForceMergeAction.java | 4 +- .../xpack/core/ilm/ForceMergeStep.java | 6 +- .../xpack/core/ilm/FreezeStep.java | 6 +- .../core/ilm/GenerateSnapshotNameStep.java | 176 ++ .../ilm/IndexLifecycleExplainResponse.java | 52 +- .../core/ilm/IndexLifecycleMetadata.java | 14 +- .../core/ilm/InitializePolicyContextStep.java | 26 +- .../core/ilm/LifecycleExecutionState.java | 61 +- .../xpack/core/ilm/LifecyclePolicy.java | 2 +- .../xpack/core/ilm/MountSnapshotStep.java | 119 ++ .../xpack/core/ilm/OpenFollowerIndexStep.java | 8 +- .../xpack/core/ilm/OpenIndexStep.java | 8 +- .../elasticsearch/xpack/core/ilm/Phase.java | 20 +- .../xpack/core/ilm/ReadOnlyAction.java | 4 +- .../xpack/core/ilm/RolloverStep.java | 20 +- .../core/ilm/SearchableSnapshotAction.java | 137 ++ .../xpack/core/ilm/SegmentCountStep.java | 12 +- .../xpack/core/ilm/SetPriorityAction.java | 6 +- .../core/ilm/SetSingleNodeAllocateStep.java | 10 +- .../xpack/core/ilm/ShrinkAction.java | 9 +- .../xpack/core/ilm/ShrinkSetAliasStep.java | 29 +- .../xpack/core/ilm/ShrinkStep.java | 20 +- .../core/ilm/ShrunkShardsAllocatedStep.java | 4 +- .../core/ilm/ShrunkenIndexCheckStep.java | 10 +- .../SwapAliasesAndDeleteSourceIndexStep.java | 121 ++ .../core/ilm/TimeseriesLifecycleType.java | 7 +- .../ilm/UpdateRolloverLifecycleDateStep.java | 24 +- .../xpack/core/ilm/UpdateSettingsStep.java | 6 +- .../core/ilm/WaitForActiveShardsStep.java | 24 +- .../core/ilm/WaitForFollowShardTasksStep.java | 8 +- .../xpack/core/ilm/WaitForIndexColorStep.java | 40 +- .../core/ilm/WaitForIndexingCompleteStep.java | 4 +- .../core/ilm/WaitForNoFollowersStep.java | 8 +- .../core/ilm/WaitForRolloverReadyStep.java | 26 +- .../xpack/core/ilm/WaitForSnapshotStep.java | 10 +- .../xpack/core/ml/MlMetadata.java | 16 +- .../elasticsearch/xpack/core/ml/MlTasks.java | 58 +- .../GetDataFrameAnalyticsStatsAction.java | 33 +- .../core/ml/action/GetFiltersAction.java | 29 +- .../action/GetTrainedModelsStatsAction.java | 40 +- .../ml/action/InternalInferModelAction.java | 47 +- .../xpack/core/ml/action/OpenJobAction.java | 4 +- .../action/StartDataFrameAnalyticsAction.java | 4 +- .../core/ml/action/StartDatafeedAction.java | 4 +- .../xpack/core/ml/action/UpdateJobAction.java | 4 +- .../xpack/core/ml/annotations/Annotation.java | 2 +- .../core/ml/annotations/AnnotationIndex.java | 27 +- .../ml/annotations/AnnotationPersister.java | 67 + .../DataFrameAnalyticsTaskState.java | 4 +- .../dataframe/analyses/BoostedTreeParams.java | 24 + .../ml/dataframe/analyses/Classification.java | 1 + .../ml/dataframe/analyses/Regression.java | 1 + .../MulticlassConfusionMatrix.java | 4 +- .../evaluation/classification/Precision.java | 2 +- .../stats/classification/Hyperparameters.java | 114 +- .../ml/dataframe/stats/common/DataCounts.java | 4 + .../stats/{ => common}/MemoryUsage.java | 26 +- .../stats/regression/Hyperparameters.java | 118 +- .../MlInferenceNamedXContentProvider.java | 31 +- .../core/ml/inference/TrainedModelConfig.java | 46 +- .../persistence/InferenceIndexConstants.java | 14 +- .../trainedmodel/ClassificationConfig.java | 101 +- .../ClassificationConfigUpdate.java | 235 +++ .../trainedmodel/InferenceConfigUpdate.java | 19 + .../trainedmodel/InferenceStats.java | 249 +++ .../LenientlyParsedInferenceConfig.java | 9 + .../trainedmodel/RegressionConfig.java | 77 +- .../trainedmodel/RegressionConfigUpdate.java | 178 ++ .../StrictlyParsedInferenceConfig.java | 9 + .../core/ml/job/config/JobTaskState.java | 2 +- .../xpack/core/ml/job/messages/Messages.java | 3 +- .../persistence/ElasticsearchMappings.java | 20 +- .../autodetect/state/ModelSnapshot.java | 4 + .../ml/job/results/CategoryDefinition.java | 65 +- .../ml/job/results/ReservedFieldNames.java | 2 + .../xpack/core/ml/utils/MlIndexAndAlias.java | 6 +- .../xpack/core/ml/utils/MlStrings.java | 4 +- .../xpack/core/ml/utils/NameResolver.java | 4 +- .../ml/utils/NamedXContentObjectHelper.java | 10 + .../xpack/core/rollup/RollupField.java | 2 +- .../rollup/action/GetRollupCapsAction.java | 4 +- .../rollup/action/GetRollupJobsAction.java | 8 +- .../action/SubmitAsyncSearchRequest.java | 48 +- .../MountSearchableSnapshotAction.java | 20 + .../MountSearchableSnapshotRequest.java | 189 ++ .../SearchableSnapshotShardStats.java | 452 ++++ .../action/realm/ClearRealmCacheRequest.java | 4 +- .../action/role/ClearRolesCacheRequest.java | 4 +- .../core/security/authc/Authentication.java | 4 + ...{TokenMetaData.java => TokenMetadata.java} | 10 +- .../authc/ldap/LdapRealmSettings.java | 4 +- ...java => LdapMetadataResolverSettings.java} | 8 +- .../security/authz/AuthorizationEngine.java | 8 +- .../authz/permission/IndicesPermission.java | 14 +- .../authz/permission/LimitedRole.java | 4 +- .../core/security/authz/permission/Role.java | 4 +- .../privilege/ClusterPrivilegeResolver.java | 9 +- .../ManageOwnApiKeyClusterPrivilege.java | 6 +- .../authz/store/ReservedRolesStore.java | 8 +- .../user/InternalUserSerializationHelper.java | 5 + .../core/slm/SnapshotLifecycleMetadata.java | 14 +- .../core/slm/SnapshotLifecyclePolicy.java | 74 +- .../slm/history/SnapshotHistoryStore.java | 12 +- .../SnapshotLifecycleTemplateRegistry.java | 4 +- .../core/ssl/TLSLicenseBootstrapCheck.java | 2 +- .../core/template/IndexTemplateRegistry.java | 6 +- .../xpack/core/template/TemplateUtils.java | 14 +- .../transform/TransformFeatureSetUsage.java | 4 +- .../action/GetTransformStatsAction.java | 4 +- .../core/transform/transforms/SyncConfig.java | 2 + .../transform/transforms/TimeSyncConfig.java | 30 +- .../pivot/DateHistogramGroupSource.java | 54 +- .../pivot/HistogramGroupSource.java | 6 +- .../transforms/pivot/SingleGroupSource.java | 6 +- .../transforms/pivot/TermsGroupSource.java | 11 +- .../transform/utils/TransformStrings.java | 4 +- .../xpack/core/upgrade/UpgradeField.java | 6 +- ...cherMetaData.java => WatcherMetadata.java} | 22 +- .../actions/stats/WatcherStatsRequest.java | 6 +- .../actions/stats/WatcherStatsResponse.java | 18 +- .../results_index_mappings.json | 6 + .../core/ml/inference_index_template.json | 5 +- .../xpack/core/ml/stats_index_mappings.json | 15 + .../idp/saml-service-provider-template.json | 3 +- .../AbstractLicenseServiceTestCase.java | 8 +- .../AbstractLicensesIntegrationTestCase.java | 20 +- .../license/LicenseClusterChangeTests.java | 22 +- .../license/LicenseRegistrationTests.java | 66 +- .../license/LicenseServiceTests.java | 4 +- .../license/LicensesManagerServiceTests.java | 26 +- ...> LicensesMetadataSerializationTests.java} | 84 +- .../RemoteClusterLicenseCheckerTests.java | 12 +- .../org/elasticsearch/license/TestUtils.java | 6 +- .../license/XPackLicenseStateTests.java | 6 +- .../snapshots/SourceOnlySnapshotIT.java | 4 +- .../SourceOnlySnapshotShardTests.java | 52 +- .../core/LocalStateCompositeXPackPlugin.java | 8 +- .../xpack/core/XPackPluginTests.java | 4 +- .../validation/SourceDestValidatorTests.java | 54 +- .../DeprecationInfoActionResponseTests.java | 10 +- .../AbstractStepMasterTimeoutTestCase.java | 12 +- .../AbstractUnfollowIndexStepTestCase.java | 4 +- .../xpack/core/ilm/AllocateActionTests.java | 10 +- .../core/ilm/AllocationRoutedStepTests.java | 114 +- .../ilm/AsyncActionBranchingStepTests.java | 145 ++ .../xpack/core/ilm/BranchingStepTests.java | 16 +- .../core/ilm/CheckShrinkReadyStepTests.java | 94 +- .../core/ilm/CleanupSnapshotStepTests.java | 156 ++ .../core/ilm/CloseFollowerIndexStepTests.java | 14 +- .../xpack/core/ilm/CloseIndexStepTests.java | 16 +- .../core/ilm/CopyExecutionStateStepTests.java | 56 +- .../xpack/core/ilm/CopySettingsStepTests.java | 79 + .../core/ilm/CreateSnapshotStepTests.java | 171 ++ .../xpack/core/ilm/DeleteActionTests.java | 45 +- .../xpack/core/ilm/DeleteStepTests.java | 18 +- .../xpack/core/ilm/ForceMergeActionTests.java | 6 +- .../xpack/core/ilm/ForceMergeStepTests.java | 12 +- .../xpack/core/ilm/FreezeStepTests.java | 20 +- .../ilm/GenerateSnapshotNameStepTests.java | 127 ++ .../IndexLifecycleExplainResponseTests.java | 19 +- .../ilm/InitializePolicyContextStepTests.java | 22 +- .../ilm/LifecycleExecutionStateTests.java | 16 +- .../ilm/LifecyclePolicyMetadataTests.java | 3 + .../xpack/core/ilm/LifecyclePolicyTests.java | 11 +- .../core/ilm/MountSnapshotStepTests.java | 253 +++ .../core/ilm/OpenFollowerIndexStepTests.java | 18 +- .../xpack/core/ilm/OpenIndexStepTests.java | 18 +- .../core/ilm/PauseFollowerIndexStepTests.java | 6 +- .../xpack/core/ilm/ReadOnlyActionTests.java | 4 +- .../xpack/core/ilm/RolloverStepTests.java | 46 +- .../ilm/SearchableSnapshotActionTests.java | 76 + .../xpack/core/ilm/SegmentCountStepTests.java | 12 +- .../core/ilm/SetPriorityActionTests.java | 8 +- .../ilm/SetSingleNodeAllocateStepTests.java | 148 +- .../xpack/core/ilm/ShrinkActionTests.java | 20 +- .../core/ilm/ShrinkSetAliasStepTests.java | 30 +- .../xpack/core/ilm/ShrinkStepTests.java | 32 +- .../ilm/ShrunkShardsAllocatedStepTests.java | 36 +- .../core/ilm/ShrunkenIndexCheckStepTests.java | 46 +- ...pAliasesAndDeleteSourceIndexStepTests.java | 135 ++ .../ilm/TimeseriesLifecycleTypeTests.java | 3 + .../ilm/UnfollowFollowIndexStepTests.java | 8 +- .../UpdateRolloverLifecycleDateStepTests.java | 50 +- .../core/ilm/UpdateSettingsStepTests.java | 18 +- .../core/ilm/WaitForActiveShardsTests.java | 50 +- .../ilm/WaitForFollowShardTasksStepTests.java | 8 +- .../core/ilm/WaitForIndexColorStepTests.java | 40 +- .../ilm/WaitForIndexingCompleteStepTests.java | 18 +- .../core/ilm/WaitForNoFollowersStepTests.java | 18 +- .../ilm/WaitForRolloverReadyStepTests.java | 58 +- .../core/ilm/WaitForSnapshotStepTests.java | 58 +- .../core/ilm/WaitForYellowStepTests.java | 16 +- .../ilm/action/PutLifecycleRequestTests.java | 6 +- .../ml/AbstractBWCSerializationTestCase.java | 12 +- .../AbstractBWCWireSerializationTestCase.java | 73 + .../xpack/core/ml/MlTasksTests.java | 86 +- ...FrameAnalyticsStatsActionRequestTests.java | 34 + ...rameAnalyticsStatsActionResponseTests.java | 25 +- .../GetDatafeedStatsActionRequestTests.java | 4 +- .../GetDatafeedsActionRequestTests.java | 4 +- .../action/GetFiltersActionRequestTests.java | 15 +- .../action/GetJobStatsActionRequestTests.java | 4 +- .../ml/action/GetJobsActionRequestTests.java | 4 +- ...TrainedModelsStatsActionResponseTests.java | 26 +- .../InternalInferModelActionRequestTests.java | 43 +- .../annotations/AnnotationPersisterTests.java | 119 ++ .../core/ml/annotations/AnnotationTests.java | 4 + .../core/ml/datafeed/DatafeedConfigTests.java | 6 +- .../DataFrameAnalyticsConfigTests.java | 3 +- .../analyses/ClassificationTests.java | 17 +- .../dataframe/analyses/RegressionTests.java | 15 +- .../classification/HyperparametersTests.java | 6 +- .../stats/{ => common}/MemoryUsageTests.java | 11 +- .../regression/HyperparametersTests.java | 6 +- .../ml/inference/TrainedModelConfigTests.java | 26 +- .../TrainedModelDefinitionTests.java | 9 +- .../ClassificationConfigTests.java | 45 +- .../ClassificationConfigUpdateTests.java | 69 + .../trainedmodel/InferenceStatsTests.java | 57 + .../trainedmodel/RegressionConfigTests.java | 41 +- .../RegressionConfigUpdateTests.java | 62 + .../trainedmodel/ensemble/EnsembleTests.java | 5 +- .../trainedmodel/tree/TreeTests.java | 15 +- .../ElasticsearchMappingsTests.java | 40 +- .../core/ml/utils/MlIndexAndAliasTests.java | 40 +- .../SearchableSnapshotShardStatsTests.java | 60 + ...DataTests.java => TokenMetadataTests.java} | 22 +- .../support/mapper/TemplateRoleNameTests.java | 20 +- .../DocumentSubsetBitsetCacheTests.java | 2 +- .../accesscontrol/FieldSubsetReaderTests.java | 30 +- ...ityIndexReaderWrapperIntegrationTests.java | 4 +- .../authz/permission/LimitedRoleTests.java | 40 +- .../ManageOwnApiKeyClusterPrivilegeTests.java | 24 +- .../authz/store/ReservedRolesStoreTests.java | 56 +- .../history/SnapshotHistoryStoreTests.java | 51 +- ...napshotLifecycleTemplateRegistryTests.java | 10 +- .../ssl/TLSLicenseBootstrapCheckTests.java | 10 +- .../GetTransformActionRequestTests.java | 4 +- .../GetTransformStatsActionRequestTests.java | 4 +- .../MockDeprecatedAggregationBuilder.java | 35 +- .../TransformConfigUpdateTests.java | 7 +- .../pivot/DateHistogramGroupSourceTests.java | 69 +- .../SnapshotRetentionConfigurationTests.java | 2 +- .../xpack/deprecation/DeprecationChecks.java | 4 +- .../deprecation/IndexDeprecationChecks.java | 20 +- .../IndexDeprecationChecksTests.java | 14 +- x-pack/plugin/enrich/build.gradle | 1 + .../xpack/enrich/EnrichMetadata.java | 8 +- .../xpack/enrich/EnrichPlugin.java | 10 +- .../EnrichPolicyMaintenanceService.java | 12 +- .../enrich/EnrichPolicyReindexPipeline.java | 2 +- .../xpack/enrich/EnrichPolicyRunner.java | 12 +- .../xpack/enrich/EnrichProcessorFactory.java | 20 +- .../xpack/enrich/EnrichStore.java | 20 +- .../action/EnrichCoordinatorStatsAction.java | 4 +- .../action/EnrichShardMultiSearchAction.java | 2 +- .../xpack/enrich/BasicEnrichTests.java | 4 +- .../xpack/enrich/EnrichPolicyRunnerTests.java | 4 +- .../enrich/EnrichProcessorFactoryTests.java | 32 +- .../xpack/enrich/GeoMatchProcessorTests.java | 2 +- .../xpack/enrich/MatchProcessorTests.java | 2 +- .../action/EnrichStatsResponseTests.java | 2 +- .../enrich/EnrichStatsCollectorTests.java | 4 +- .../test/eql/CommonEqlActionTestCase.java | 9 + .../src/main/resources/mapping-default.json | 80 + .../src/main/resources/test_queries.toml | 9 +- .../resources/test_queries_supported.toml | 15 + .../resources/test_queries_unsupported.toml | 103 +- x-pack/plugin/eql/src/main/antlr/EqlBase.g4 | 4 +- .../plugin/eql/src/main/antlr/EqlBase.tokens | 77 + .../eql/src/main/antlr/EqlBaseLexer.tokens | 77 + .../xpack/eql/action/EqlSearchRequest.java | 16 + .../xpack/eql/action/EqlSearchTask.java | 33 + .../xpack/eql/execution/search/Querier.java | 7 +- .../function/EqlFunctionRegistry.java | 18 +- .../function/scalar/string/Between.java | 155 ++ .../scalar/string/BetweenFunctionPipe.java | 129 ++ .../string/BetweenFunctionProcessor.java | 120 ++ .../function/scalar/string/CIDRMatch.java | 114 + .../function/scalar/string/EndsWith.java | 120 ++ .../scalar/string/EndsWithFunctionPipe.java | 104 + .../string/EndsWithFunctionProcessor.java | 95 + .../function/scalar/string/Length.java | 106 + .../scalar/string/LengthFunctionPipe.java | 94 + .../string/LengthFunctionProcessor.java | 78 + .../function/scalar/string/StartsWith.java | 120 ++ .../scalar/string/StartsWithFunctionPipe.java | 104 + .../string/StartsWithFunctionProcessor.java | 95 + .../scalar/string/StringContains.java | 116 ++ .../string/StringContainsFunctionPipe.java | 106 + .../StringContainsFunctionProcessor.java | 96 + .../function/scalar/string/StringUtils.java | 65 +- .../function/scalar/string/Substring.java | 2 +- .../function/scalar/string/Wildcard.java | 106 + .../whitelist/InternalEqlScriptUtils.java | 25 + .../xpack/eql/optimizer/Optimizer.java | 23 +- .../xpack/eql/parser/EqlBaseLexer.java | 38 +- .../xpack/eql/parser/EqlBaseParser.java | 84 +- .../xpack/eql/parser/EqlParser.java | 67 +- .../xpack/eql/parser/ExpressionBuilder.java | 18 +- .../xpack/eql/parser/LogicalPlanBuilder.java | 169 +- .../xpack/eql/plan/logical/Join.java | 97 + .../xpack/eql/plan/logical/KeyedFilter.java | 70 + .../xpack/eql/plan/logical/Sequence.java | 72 + .../xpack/eql/plugin/EqlStatsRequest.java | 18 +- .../eql/plugin/TransportEqlSearchAction.java | 7 +- .../xpack/eql/session/Configuration.java | 13 +- .../xpack/eql/session/EqlSession.java | 11 +- .../xpack/eql/util/StringUtils.java | 29 + .../xpack/eql/plugin/eql_whitelist.txt | 5 + .../elasticsearch/xpack/eql/EqlTestUtils.java | 15 +- .../eql/action/EqlSearchResponseTests.java | 2 +- .../xpack/eql/analysis/CancellationTests.java | 125 ++ .../xpack/eql/analysis/VerifierTests.java | 46 +- .../string/BetweenFunctionProcessorTests.java | 37 + .../scalar/string/EndsWithProcessorTests.java | 54 + .../scalar/string/LengthProcessorTests.java | 41 + .../string/StartsWithProcessorTests.java | 54 + .../StringContainsFunctionProcessorTests.java | 49 + .../scalar/string/StringUtilsTests.java | 78 + .../xpack/eql/parser/ExpressionTests.java | 49 +- .../xpack/eql/parser/LogicalPlanTests.java | 70 +- .../planner/AbstractQueryFolderTestCase.java | 2 +- .../eql/planner/QueryFolderFailTests.java | 155 +- .../xpack/eql/planner/QueryFolderOkTests.java | 81 +- .../src/test/resources/mapping-default.json | 12 + .../src/test/resources/queries-supported.eql | 235 +++ .../test/resources/queries-unsupported.eql | 287 +-- .../src/test/resources/queryfolder_tests.txt | 154 +- .../FrozenIndicesUsageTransportAction.java | 6 +- .../action/TransportFreezeIndexAction.java | 30 +- .../engine/FrozenIndexRecoveryTests.java | 6 +- .../index/engine/FrozenIndexShardTests.java | 2 +- .../index/engine/FrozenIndexTests.java | 34 +- .../xpack/graph/test/GraphTests.java | 4 +- x-pack/plugin/identity-provider/build.gradle | 2 +- .../qa/idp-rest-tests/build.gradle | 19 +- .../idp/IdentityProviderAuthenticationIT.java | 194 ++ .../xpack/idp/IdpRestTestCase.java | 144 +- .../idp/ManageServiceProviderRestIT.java | 70 +- .../idp/WildcardServiceProviderRestIT.java | 124 ++ .../src/test/resources/idp-metadata.xml | 32 + .../src/test/resources/roles.yml | 7 +- .../src/test/resources/wildcard_services.json | 41 + .../xpack/idp/IdentityProviderPlugin.java | 32 +- .../SamlInitiateSingleSignOnRequest.java | 28 +- .../SamlInitiateSingleSignOnResponse.java | 26 +- .../xpack/idp/action/SamlMetadataRequest.java | 23 +- .../SamlValidateAuthnRequestResponse.java | 11 +- ...TransportPutSamlServiceProviderAction.java | 3 + ...ansportSamlInitiateSingleSignOnAction.java | 110 +- .../action/TransportSamlMetadataAction.java | 3 +- .../ApplicationActionsResolver.java | 135 ++ .../privileges/ServiceProviderPrivileges.java | 15 +- .../idp/privileges/UserPrivilegeResolver.java | 87 +- .../saml/authn/SamlAuthnRequestValidator.java | 19 +- .../idp/saml/idp/SamlIdentityProvider.java | 45 +- .../saml/idp/SamlIdentityProviderBuilder.java | 8 +- .../idp/saml/idp/SamlMetadataGenerator.java | 4 +- .../RestSamlInitiateSingleSignOnAction.java | 3 + .../rest/action/RestSamlMetadataAction.java | 3 +- ...mlValidateAuthenticationRequestAction.java | 1 + .../saml/sp/SamlServiceProviderDocument.java | 32 +- .../saml/sp/SamlServiceProviderFactory.java | 90 + .../idp/saml/sp/SamlServiceProviderIndex.java | 10 +- .../saml/sp/SamlServiceProviderResolver.java | 81 +- .../saml/sp/ServiceProviderCacheSettings.java | 39 + .../idp/saml/sp/WildcardServiceProvider.java | 198 ++ .../sp/WildcardServiceProviderResolver.java | 232 +++ .../saml/support/SamlAuthenticationState.java | 46 +- .../PutSamlServiceProviderRequestTests.java | 6 +- .../idp/action/SamlIdentityProviderTests.java | 134 +- .../SamlInitiateSingleSignOnRequestTests.java | 8 +- ...tSamlInitiateSingleSignOnActionTests.java} | 47 +- .../UserPrivilegeResolverTests.java | 49 +- .../authn/SamlAuthnRequestValidatorTests.java | 10 +- .../idp/SamlIdentityProviderBuilderTests.java | 47 +- .../saml/idp/SamlMetadataGeneratorTests.java | 8 +- .../sp/SamlServiceProviderDocumentTests.java | 15 +- .../sp/SamlServiceProviderIndexTests.java | 53 +- .../sp/SamlServiceProviderResolverTests.java | 11 +- .../WildcardServiceProviderResolverTests.java | 182 ++ .../support/SamlAuthenticationStateTests.java | 21 - .../test/IdentityProviderIntegTestCase.java | 4 +- .../xpack/idp/saml/test/IdpSamlTestCase.java | 9 +- .../xpack/ilm/ChangePolicyforIndexIT.java | 6 +- .../ilm/TimeSeriesLifecycleActionsIT.java | 403 ++-- .../xpack/slm/SnapshotLifecycleRestIT.java | 1 + .../xpack/ilm/ExecuteStepsUpdateTask.java | 22 +- .../xpack/ilm/IndexLifecycle.java | 12 +- .../xpack/ilm/IndexLifecycleRunner.java | 226 +- .../xpack/ilm/IndexLifecycleService.java | 35 +- .../xpack/ilm/IndexLifecycleTransition.java | 56 +- .../IndexLifecycleUsageTransportAction.java | 8 +- .../xpack/ilm/MoveToErrorStepUpdateTask.java | 8 +- .../xpack/ilm/MoveToNextStepUpdateTask.java | 10 +- .../xpack/ilm/OperationModeUpdateTask.java | 30 +- .../xpack/ilm/PolicyStepsRegistry.java | 12 +- .../xpack/ilm/SetStepInfoUpdateTask.java | 4 +- .../TransportDeleteLifecycleAction.java | 10 +- .../TransportExplainLifecycleAction.java | 6 +- .../action/TransportGetLifecycleAction.java | 2 +- .../ilm/action/TransportGetStatusAction.java | 2 +- .../ilm/action/TransportMoveToStepAction.java | 16 +- .../action/TransportPutLifecycleAction.java | 24 +- .../ilm/action/TransportRetryAction.java | 4 +- .../ilm/action/TransportStopILMAction.java | 4 +- .../xpack/ilm/history/ILMHistoryStore.java | 12 +- .../elasticsearch/xpack/ilm/package-info.java | 6 +- .../xpack/slm/SLMUsageTransportAction.java | 2 +- .../xpack/slm/SnapshotLifecycleService.java | 12 +- .../xpack/slm/SnapshotLifecycleTask.java | 10 +- .../xpack/slm/SnapshotRetentionTask.java | 4 +- .../slm/UpdateSnapshotLifecycleStatsTask.java | 6 +- ...ransportDeleteSnapshotLifecycleAction.java | 8 +- ...ansportExecuteSnapshotLifecycleAction.java | 2 +- .../action/TransportGetSLMStatusAction.java | 2 +- .../TransportGetSnapshotLifecycleAction.java | 2 +- ...nsportGetSnapshotLifecycleStatsAction.java | 2 +- .../TransportPutSnapshotLifecycleAction.java | 8 +- .../ilm/ExecuteStepsUpdateTaskTests.java | 58 +- ...ndexLifecycleInfoTransportActionTests.java | 16 +- .../IndexLifecycleInitialisationTests.java | 12 +- .../ilm/IndexLifecycleMetadataTests.java | 20 +- .../xpack/ilm/IndexLifecycleRunnerTests.java | 214 +- .../xpack/ilm/IndexLifecycleServiceTests.java | 91 +- .../ilm/IndexLifecycleTransitionTests.java | 112 +- .../ilm/MoveToErrorStepUpdateTaskTests.java | 22 +- .../ilm/MoveToNextStepUpdateTaskTests.java | 24 +- .../ilm/OperationModeUpdateTaskTests.java | 16 +- .../xpack/ilm/PolicyStepsRegistryTests.java | 68 +- .../xpack/ilm/SetStepInfoUpdateTaskTests.java | 22 +- .../xpack/ilm/UpdateSettingsStepTests.java | 8 +- .../TransportPutLifecycleActionTests.java | 52 +- .../action/TransportStopILMActionTests.java | 82 + .../ilm/history/ILMHistoryStoreTests.java | 44 +- .../slm/SLMSnapshotBlockingIntegTests.java | 55 +- .../SnapshotLifecycleInitialisationTests.java | 2 +- .../slm/SnapshotLifecyclePolicyTests.java | 25 - .../slm/SnapshotLifecycleServiceTests.java | 18 +- .../xpack/slm/SnapshotLifecycleTaskTests.java | 16 +- .../xpack/slm/SnapshotRetentionTaskTests.java | 12 +- .../xpack/logstash/Logstash.java | 7 +- .../mapper/ConstantKeywordFieldMapper.java | 6 + .../mapper/FlatObjectFieldMapper.java | 9 +- .../ml/qa/ml-with-security/build.gradle | 5 +- .../ml/integration/BulkFailureRetryIT.java | 6 +- .../ml/integration/CategorizationIT.java | 85 + .../ClassificationEvaluationIT.java | 489 +++-- .../ml/integration/ClassificationIT.java | 25 +- .../ml/integration/DatafeedJobsRestIT.java | 49 + .../ExplainDataFrameAnalyticsIT.java | 41 + .../xpack/ml/integration/ForecastIT.java | 8 +- .../ml/integration/InferenceIngestIT.java | 48 +- .../MlNativeAutodetectIntegTestCase.java | 6 - ...NativeDataFrameAnalyticsIntegTestCase.java | 3 +- .../ml/integration/MlNativeIntegTestCase.java | 19 +- .../xpack/ml/integration/RegressionIT.java | 4 +- .../ml/integration/ScheduledEventsIT.java | 4 +- .../ml/integration/SetUpgradeModeIT.java | 8 +- .../xpack/ml/integration/TrainedModelIT.java | 2 + .../ml/transforms/PainlessDomainSplitIT.java | 10 +- .../xpack/ml/MachineLearning.java | 32 +- .../MachineLearningUsageTransportAction.java | 9 +- .../xpack/ml/MlAssignmentNotifier.java | 16 +- .../ml/MlConfigMigrationEligibilityCheck.java | 8 +- .../xpack/ml/MlConfigMigrator.java | 40 +- .../xpack/ml/MlDailyMaintenanceService.java | 9 +- .../xpack/ml/MlInitializationService.java | 2 +- .../ml/action/TransportCloseJobAction.java | 48 +- ...ansportDeleteDataFrameAnalyticsAction.java | 4 +- .../action/TransportDeleteDatafeedAction.java | 12 +- .../action/TransportDeleteForecastAction.java | 6 +- .../ml/action/TransportDeleteJobAction.java | 22 +- .../TransportDeleteTrainedModelAction.java | 2 +- .../TransportEstimateModelMemoryAction.java | 176 +- ...sportGetDataFrameAnalyticsStatsAction.java | 11 +- .../TransportGetDatafeedsStatsAction.java | 8 +- .../action/TransportGetJobsStatsAction.java | 12 +- .../TransportGetTrainedModelsStatsAction.java | 24 +- .../TransportInternalInferModelAction.java | 8 +- .../TransportIsolateDatafeedAction.java | 6 +- .../ml/action/TransportJobTaskAction.java | 6 +- .../ml/action/TransportKillProcessAction.java | 6 +- .../ml/action/TransportOpenJobAction.java | 40 +- .../TransportPreviewDatafeedAction.java | 72 +- .../TransportPutDataFrameAnalyticsAction.java | 46 +- .../ml/action/TransportPutDatafeedAction.java | 77 +- .../TransportPutTrainedModelAction.java | 16 + .../TransportRevertModelSnapshotAction.java | 4 +- .../action/TransportSetUpgradeModeAction.java | 34 +- ...ransportStartDataFrameAnalyticsAction.java | 36 +- .../action/TransportStartDatafeedAction.java | 42 +- ...TransportStopDataFrameAnalyticsAction.java | 246 ++- .../action/TransportStopDatafeedAction.java | 22 +- .../action/TransportUpdateDatafeedAction.java | 37 +- .../xpack/ml/datafeed/DatafeedJob.java | 84 +- .../xpack/ml/datafeed/DatafeedJobBuilder.java | 19 +- .../xpack/ml/datafeed/DatafeedManager.java | 20 +- .../ml/datafeed/DatafeedNodeSelector.java | 16 +- .../RollupDataExtractorFactory.java | 8 +- .../persistence/DatafeedConfigProvider.java | 12 +- .../dataframe/DataFrameAnalyticsManager.java | 11 +- .../ml/dataframe/DataFrameAnalyticsTask.java | 15 +- .../xpack/ml/dataframe/DestinationIndex.java | 20 +- .../xpack/ml/dataframe/MappingsMerger.java | 18 +- .../process/AnalyticsResultProcessor.java | 31 +- .../process/results/AnalyticsResult.java | 2 +- .../xpack/ml/dataframe/stats/StatsHolder.java | 2 +- .../inference/TrainedModelStatsService.java | 180 ++ .../inference/ingest/InferenceProcessor.java | 27 +- .../inference/loadingservice/LocalModel.java | 66 +- .../ml/inference/loadingservice/Model.java | 8 +- .../loadingservice/ModelLoadingService.java | 116 +- .../persistence/TrainedModelProvider.java | 103 +- .../xpack/ml/job/JobManager.java | 24 +- .../xpack/ml/job/JobNodeSelector.java | 28 +- .../job/persistence/CalendarQueryBuilder.java | 4 +- .../ml/job/persistence/JobConfigProvider.java | 18 +- .../ml/job/persistence/JobDataDeleter.java | 3 + .../job/persistence/JobResultsProvider.java | 14 +- .../autodetect/AutodetectCommunicator.java | 16 +- .../autodetect/AutodetectProcessManager.java | 48 +- .../output/AutodetectResultProcessor.java | 36 +- .../NativeNormalizerProcessFactory.java | 10 +- .../xpack/ml/process/MlMemoryTracker.java | 18 +- .../cat/RestCatDataFrameAnalyticsAction.java | 4 +- .../xpack/ml/rest/cat/RestCatJobsAction.java | 4 +- .../rest/cat/RestCatTrainedModelsAction.java | 4 +- .../ml/rest/filter/RestGetFiltersAction.java | 8 +- .../inference/RestGetTrainedModelsAction.java | 4 +- .../RestGetTrainedModelsStatsAction.java | 4 +- .../ml/rest/job/RestDeleteForecastAction.java | 4 +- .../ml/rest/job/RestGetJobStatsAction.java | 4 +- .../xpack/ml/rest/job/RestGetJobsAction.java | 4 +- .../ml/utils/SecondaryAuthorizationUtils.java | 31 + ...s.java => MachineLearningLicensingIT.java} | 22 +- ...chineLearningInfoTransportActionTests.java | 6 +- .../xpack/ml/MlAssignmentNotifierTests.java | 38 +- ...lConfigMigrationEligibilityCheckTests.java | 102 +- .../xpack/ml/MlConfigMigratorTests.java | 48 +- .../ml/MlDailyMaintenanceServiceTests.java | 37 +- .../action/TransportCloseJobActionTests.java | 28 +- ...ansportEstimateModelMemoryActionTests.java | 74 +- ...sportGetTrainedModelsStatsActionTests.java | 12 +- .../action/TransportOpenJobActionTests.java | 52 +- ...ortStartDataFrameAnalyticsActionTests.java | 24 +- .../TransportStartDatafeedActionTests.java | 14 +- ...portStopDataFrameAnalyticsActionTests.java | 59 +- .../TransportStopDatafeedActionTests.java | 14 +- .../ml/datafeed/DatafeedJobBuilderTests.java | 9 +- .../xpack/ml/datafeed/DatafeedJobTests.java | 5 +- .../ml/datafeed/DatafeedManagerTests.java | 60 +- .../datafeed/DatafeedNodeSelectorTests.java | 70 +- .../ml/dataframe/DestinationIndexTests.java | 38 +- .../ml/dataframe/MappingsMergerTests.java | 44 +- .../AnalyticsResultProcessorTests.java | 12 +- .../process/DataFrameRowsJoinerTests.java | 2 +- ...tratifiedCrossValidationSplitterTests.java | 6 +- .../process/results/AnalyticsResultTests.java | 4 +- .../InferenceProcessorFactoryTests.java | 18 +- .../ingest/InferenceProcessorTests.java | 31 +- .../loadingservice/LocalModelTests.java | 134 +- .../ModelLoadingServiceTests.java | 92 +- .../ml/integration/AnnotationIndexIT.java | 10 +- .../AutodetectResultProcessorIT.java | 93 +- .../integration/BasicDistributedJobsIT.java | 16 +- .../integration/DatafeedConfigProviderIT.java | 8 +- .../ml/integration/EstablishedMemUsageIT.java | 2 +- .../ml/integration/JobConfigProviderIT.java | 8 +- .../ml/integration/JobResultsProviderIT.java | 24 +- .../integration/JobStorageDeletionTaskIT.java | 6 +- .../ml/integration/MlConfigMigratorIT.java | 56 +- .../integration/MlDistributedFailureIT.java | 16 +- .../xpack/ml/integration/MlFiltersIT.java | 36 + .../integration/ModelInferenceActionIT.java | 24 +- .../xpack/ml/integration/TooManyJobsIT.java | 12 +- .../xpack/ml/job/JobManagerTests.java | 62 +- .../xpack/ml/job/JobNodeSelectorTests.java | 146 +- .../persistence/JobResultsPersisterTests.java | 2 +- .../persistence/JobResultsProviderTests.java | 32 +- .../AutodetectCommunicatorTests.java | 6 +- .../AutodetectProcessManagerTests.java | 35 +- .../AutodetectResultProcessorTests.java | 32 +- .../job/results/CategoryDefinitionTests.java | 22 +- .../ml/process/MlMemoryTrackerTests.java | 34 +- .../ml/process/ProcessResultsParserTests.java | 3 +- .../xpack/ml/support/BaseMlIntegTestCase.java | 10 +- .../ResultsPersisterServiceTests.java | 2 +- x-pack/plugin/monitoring/build.gradle | 3 - .../action/TransportMonitoringBulkAction.java | 2 +- .../xpack/monitoring/collector/Collector.java | 2 +- .../cluster/ClusterStatsMonitoringDoc.java | 8 +- .../indices/IndexStatsCollector.java | 4 +- .../indices/IndexStatsMonitoringDoc.java | 26 +- .../collector/ml/JobStatsCollector.java | 4 +- .../collector/node/NodeStatsCollector.java | 11 +- .../exporter/ClusterAlertsUtil.java | 6 +- .../xpack/monitoring/exporter/Exporters.java | 2 +- .../exporter/http/HttpExporter.java | 2 +- .../exporter/http/TemplateHttpResource.java | 4 +- .../exporter/local/LocalExporter.java | 12 +- .../monitoring/BaseCollectorTestCase.java | 10 +- .../TransportMonitoringBulkActionTests.java | 8 +- .../local/LocalIndicesCleanerTests.java | 4 +- .../cluster/ClusterStatsCollectorTests.java | 4 +- .../ClusterStatsMonitoringDocTests.java | 7 +- .../indices/IndexRecoveryCollectorTests.java | 4 +- .../indices/IndexStatsCollectorTests.java | 20 +- .../indices/IndexStatsMonitoringDocTests.java | 40 +- .../IndicesStatsMonitoringDocTests.java | 8 +- .../collector/ml/JobStatsCollectorTests.java | 8 +- .../node/NodeStatsCollectorTests.java | 4 +- .../node/NodeStatsMonitoringDocTests.java | 2 +- .../shards/ShardsCollectorTests.java | 4 +- .../exporter/ClusterAlertsUtilTests.java | 8 +- .../monitoring/exporter/ExportersTests.java | 6 +- .../exporter/http/HttpExporterIT.java | 6 +- .../http/HttpExporterResourceTests.java | 8 +- .../exporter/http/HttpExporterTests.java | 6 +- .../local/LocalExporterIntegTests.java | 6 +- .../LocalExporterResourceIntegTests.java | 4 +- .../test/MonitoringIntegTestCase.java | 4 +- .../xpack/ql/expression/Expressions.java | 17 +- .../xpack/ql/expression/TypeResolutions.java | 14 + .../expression/function/FunctionRegistry.java | 66 +- .../scalar/BaseSurrogateFunction.java | 57 + .../function/scalar/SurrogateFunction.java | 12 + .../xpack/ql/index/IndexResolver.java | 84 +- .../xpack/ql/optimizer/OptimizerRules.java | 16 + .../ql/planner/ExpressionTranslator.java | 2 +- .../ql/planner/ExpressionTranslators.java | 57 +- .../xpack/ql/planner/QlTranslatorHandler.java | 7 + .../xpack/ql/planner/TranslatorHandler.java | 5 +- .../elasticsearch/xpack/ql/util/Check.java | 12 + x-pack/plugin/rollup/build.gradle | 4 - .../rollup/RollupJobIdentifierUtils.java | 2 +- .../xpack/rollup/RollupRequestTranslator.java | 17 +- .../rollup/RollupResponseTranslator.java | 6 +- .../xpack/rollup/action/RollupIndexCaps.java | 4 +- .../TransportDeleteRollupJobAction.java | 4 +- .../action/TransportGetRollupCapsAction.java | 22 +- .../TransportGetRollupIndexCapsAction.java | 6 +- .../action/TransportGetRollupJobAction.java | 10 +- .../action/TransportPutRollupJobAction.java | 8 +- .../action/TransportRollupSearchAction.java | 16 +- .../xpack/rollup/job/RollupIndexer.java | 15 +- .../xpack/rollup/job/RollupJobTask.java | 4 +- .../rollup/RollupJobIdentifierUtilTests.java | 2 +- .../rollup/RollupRequestTranslationTests.java | 6 +- .../RollupResponseTranslationTests.java | 36 +- .../action/GetJobsActionRequestTests.java | 48 +- .../GetRollupCapsActionRequestTests.java | 60 +- .../GetRollupIndexCapsActionRequestTests.java | 34 +- .../action/PutJobStateMachineTests.java | 54 +- .../rollup/action/RollupIndexCapsTests.java | 4 +- .../rollup/action/SearchActionTests.java | 64 +- .../job/RollupIndexerIndexingTests.java | 6 +- .../rollup/job/RollupIndexerStateTests.java | 10 +- .../xpack/rollup/job/RollupJobTaskTests.java | 50 +- .../PinnedQueryBuilderTests.java | 4 +- .../plugin/searchable-snapshots/build.gradle | 42 + .../qa/azure/build.gradle | 102 + .../AzureSearchableSnapshotsIT.java | 31 + .../searchable-snapshots/qa/build.gradle | 6 + .../searchable-snapshots/qa/rest/build.gradle | 31 + .../rest/FsSearchableSnapshotsIT.java | 32 + ...rchableSnapshotsClientYamlTestSuiteIT.java | 22 + .../rest-api-spec/test/clear_cache.yml | 134 ++ .../resources/rest-api-spec/test/stats.yml | 212 ++ .../searchable-snapshots/qa/s3/build.gradle | 79 + .../s3/S3SearchableSnapshotsIT.java | 31 + .../BaseSearchableSnapshotIndexInput.java | 178 ++ .../store/InMemoryNoOpCommitDirectory.java | 125 ++ .../index/store/IndexInputStats.java | 225 ++ .../store/SearchableSnapshotDirectory.java | 393 ++++ .../index/store/cache/CacheFile.java | 290 +++ .../index/store/cache/CacheKey.java | 74 + .../cache/CachedBlobContainerIndexInput.java | 343 ++++ .../index/store/cache/SparseFileTracker.java | 326 +++ .../direct/DirectBlobContainerIndexInput.java | 399 ++++ .../SearchableSnapshotAllocator.java | 110 + .../SearchableSnapshotIndexEventListener.java | 59 + .../SearchableSnapshots.java | 275 +++ ...actTransportSearchableSnapshotsAction.java | 123 ++ .../ClearSearchableSnapshotsCacheAction.java | 18 + .../ClearSearchableSnapshotsCacheRequest.java | 27 + ...ClearSearchableSnapshotsCacheResponse.java | 29 + .../SearchableSnapshotsStatsAction.java | 18 + .../SearchableSnapshotsStatsRequest.java | 27 + .../SearchableSnapshotsStatsResponse.java | 95 + ...rtClearSearchableSnapshotsCacheAction.java | 86 + ...ransportMountSearchableSnapshotAction.java | 190 ++ ...ansportSearchableSnapshotsStatsAction.java | 124 ++ .../cache/CacheService.java | 132 ++ ...stClearSearchableSnapshotsCacheAction.java | 41 + .../RestMountSearchableSnapshotAction.java | 46 + .../RestSearchableSnapshotsStatsAction.java | 42 + .../InMemoryNoOpCommitDirectoryTests.java | 212 ++ .../index/store/IndexInputStatsTests.java | 128 ++ ...SearchableSnapshotDirectoryStatsTests.java | 605 ++++++ .../SearchableSnapshotDirectoryTests.java | 594 ++++++ .../index/store/cache/CacheFileTests.java | 144 ++ .../index/store/cache/CacheKeyTests.java | 110 + .../CachedBlobContainerIndexInputTests.java | 309 +++ .../store/cache/SparseFileTrackerTests.java | 232 +++ .../index/store/cache/TestUtils.java | 195 ++ .../DirectBlobContainerIndexInputTests.java | 237 +++ ...stractSearchableSnapshotsRestTestCase.java | 424 ++++ .../BaseSearchableSnapshotsIntegTestCase.java | 72 + .../SearchableSnapshotsIntegTests.java | 313 +++ .../SearchableSnapshotsLicenseIntegTests.java | 157 ++ .../MountSearchableSnapshotRequestTests.java | 171 ++ ...SearchableSnapshotsStatsResponseTests.java | 123 ++ x-pack/plugin/security/build.gradle | 2 +- x-pack/plugin/security/cli/build.gradle | 6 +- .../SecurityOnTrialLicenseRestTestCase.java | 2 +- .../xpack/security/TlsWithBasicLicenseIT.java | 1 + .../xpack/security/Security.java | 33 +- .../security/SecurityInfoTransportAction.java | 8 +- .../SecurityUsageTransportAction.java | 11 +- .../action/filter/SecurityActionFilter.java | 2 +- .../TransportOpenIdConnectLogoutAction.java | 2 +- .../saml/TransportSamlLogoutAction.java | 2 +- .../xpack/security/authc/ApiKeyService.java | 7 +- .../xpack/security/authc/TokenService.java | 104 +- .../ldap/ActiveDirectorySessionFactory.java | 34 +- .../xpack/security/authc/ldap/LdapRealm.java | 2 +- .../authc/ldap/LdapSessionFactory.java | 2 +- .../ldap/LdapUserSearchSessionFactory.java | 10 +- ...esolver.java => LdapMetadataResolver.java} | 10 +- .../authc/ldap/support/LdapSession.java | 18 +- .../authc/ldap/support/SessionFactory.java | 4 +- .../security/authz/AuthorizationService.java | 34 +- .../authz/IndicesAndAliasesResolver.java | 99 +- .../xpack/security/authz/RBACEngine.java | 21 +- .../DeprecationRoleDescriptorConsumer.java | 16 +- .../rest/action/SecurityBaseRestHandler.java | 2 +- .../support/SecurityIndexManager.java | 74 +- .../support/SecurityStatusChangeListener.java | 2 +- .../transport/nio/SecurityNioTransport.java | 2 +- .../DocumentAndFieldLevelSecurityTests.java | 32 +- .../integration/KibanaUserRoleIntegTests.java | 24 +- .../PermissionPrecedenceTests.java | 4 +- .../test/SecurityIntegTestCase.java | 14 +- .../SecurityInfoTransportActionTests.java | 27 +- .../xpack/security/SecurityTests.java | 24 +- .../xpack/security/TemplateUpgraderTests.java | 10 +- .../saml/TransportSamlLogoutActionTests.java | 10 +- .../action/user/PutUserRequestTests.java | 2 +- .../security/authc/ApiKeyIntegTests.java | 2 +- .../authc/AuthenticationServiceTests.java | 4 +- .../authc/SecurityRealmSettingsTests.java | 2 +- .../security/authc/TokenAuthIntegTests.java | 4 +- .../security/authc/TokenServiceTests.java | 26 +- .../authc/esnative/NativeRealmTests.java | 4 +- .../authc/ldap/ActiveDirectoryRealmTests.java | 6 +- .../security/authc/ldap/LdapRealmTests.java | 6 +- ...ts.java => LdapMetadataResolverTests.java} | 16 +- .../authc/saml/SamlRealmTestHelper.java | 2 +- .../security/authc/saml/SamlRealmTests.java | 8 +- .../mapper/NativeRoleMappingStoreTests.java | 4 +- .../authz/AuthorizationServiceTests.java | 106 +- .../authz/AuthorizedIndicesTests.java | 62 +- .../authz/IndicesAndAliasesResolverTests.java | 100 +- .../xpack/security/authz/RBACEngineTests.java | 5 +- .../security/authz/ReadActionsTests.java | 2 + ...ldDataCacheWithFieldSubsetReaderTests.java | 8 +- .../accesscontrol/IndicesPermissionTests.java | 72 +- .../accesscontrol/OptOutQueryCacheTests.java | 18 +- .../authz/store/CompositeRolesStoreTests.java | 12 +- ...eprecationRoleDescriptorConsumerTests.java | 118 +- .../authz/store/NativeRolesStoreTests.java | 22 +- .../action/SecurityBaseRestHandlerTests.java | 8 +- .../apikey/RestCreateApiKeyActionTests.java | 1 + .../apikey/RestGetApiKeyActionTests.java | 1 + .../RestInvalidateApiKeyActionTests.java | 1 + .../support/SecurityIndexManagerTests.java | 112 +- .../SecurityStatusChangeListenerTests.java | 8 +- .../security/test/SecurityTestUtils.java | 18 +- .../filter/IpFilteringUpdateTests.java | 24 +- .../security/user/UserSerializationTests.java | 11 + x-pack/plugin/spatial/build.gradle | 1 + .../xpack/spatial/SpatialPlugin.java | 2 + .../spatial/index/mapper/CartesianPoint.java | 293 +++ .../index/mapper/PointFieldMapper.java | 352 ++++ .../index/query/ShapeQueryBuilder.java | 9 +- .../index/query/ShapeQueryPointProcessor.java | 174 ++ .../mapper/CartesianFieldMapperTests.java | 175 ++ .../index/mapper/PointFieldMapperTests.java | 297 +++ .../index/mapper/PointFieldTypeTests.java | 16 + .../index/mapper/ShapeFieldMapperTests.java | 34 +- .../ShapeQueryBuilderOverPointTests.java | 49 + .../ShapeQueryBuilderOverShapeTests.java | 52 + .../index/query/ShapeQueryBuilderTests.java | 56 +- .../search/ShapeQueryOverPointTests.java | 159 ++ .../search/ShapeQueryOverShapeTests.java | 348 ++++ .../xpack/spatial/search/ShapeQueryTests.java | 519 +++-- .../xpack/sql/jdbc/ConnectionProxy.java | 2 +- ...aProxy.java => DatabaseMetaDataProxy.java} | 4 +- .../elasticsearch/xpack/sql/jdbc/Debug.java | 7 +- .../xpack/sql/jdbc/JdbcDatabaseMetaData.java | 4 +- .../xpack/sql/jdbc/JdbcParameterMetaData.java | 2 +- .../xpack/sql/jdbc/JdbcResultSet.java | 4 +- .../sql/jdbc/JdbcDatabaseMetaDataTests.java | 2 +- .../sql/jdbc/JdbcResultSetMetaDataTests.java | 6 +- .../xpack/sql/qa/security/JdbcSecurityIT.java | 20 +- .../sql/qa/security/RestSqlSecurityIT.java | 2 +- .../sql/qa/security/with-ssl/build.gradle | 9 +- .../xpack/sql/qa/jdbc/ConnectionTestCase.java | 2 +- .../sql/qa/jdbc/DatabaseMetaDataTestCase.java | 6 +- .../xpack/sql/qa/jdbc/FetchSizeTestCase.java | 18 +- .../xpack/sql/qa/jdbc/JdbcAssert.java | 16 +- .../xpack/sql/qa/jdbc/JdbcTestUtils.java | 6 +- .../qa/jdbc/ResultSetMetaDataTestCase.java | 10 +- .../xpack/sql/qa/jdbc/ShowTablesTestCase.java | 6 +- .../sql/qa/src/main/resources/alias.csv-spec | 10 +- .../qa/src/main/resources/command.csv-spec | 35 +- .../qa/src/main/resources/datetime.csv-spec | 104 + .../qa/src/main/resources/docs/docs.csv-spec | 90 +- .../qa/src/main/resources/geo/geosql.csv-spec | 4 +- .../qa/src/main/resources/ogc/ogc.csv-spec | 4 +- .../setup_mock_metadata_get_table_types.sql | 2 +- .../setup_mock_metadata_get_tables.sql | 4 +- .../setup_mock_metadata_get_tables_empty.sql | 4 +- ...setup_mock_metadata_get_types_of_table.sql | 4 +- .../src/main/resources/slow/frozen.csv-spec | 14 +- .../function/SqlFunctionRegistry.java | 4 +- .../function/scalar/Processors.java | 4 +- .../BinaryDateTimeDatePartFunction.java | 92 + .../datetime/BinaryDateTimeFunction.java | 36 +- .../function/scalar/datetime/DatePart.java | 2 +- .../scalar/datetime/DateTimeFormat.java | 70 + .../scalar/datetime/DateTimeFormatPipe.java | 36 + .../datetime/DateTimeFormatProcessor.java | 80 + .../function/scalar/datetime/DateTrunc.java | 2 +- .../whitelist/InternalSqlScriptUtils.java | 7 +- .../xpack/sql/parser/CommandBuilder.java | 22 +- .../plan/logical/command/sys/SysColumns.java | 6 +- .../plan/logical/command/sys/SysTables.java | 24 +- .../plan/logical/command/sys/SysTypes.java | 2 +- .../xpack/sql/planner/QueryTranslator.java | 27 +- .../sql/planner/SqlTranslatorHandler.java | 9 +- .../xpack/sql/plugin/SqlStatsRequest.java | 18 +- .../xpack/sql/plugin/sql_whitelist.txt | 1 + .../analyzer/VerifierErrorMessagesTests.java | 35 +- .../extractor/TestMultiValueAggregation.java | 3 +- .../extractor/TestSingleValueAggregation.java | 3 +- .../extractor/TopHitsAggExtractorTests.java | 6 +- .../datetime/DateTimeFormatPipeTests.java | 126 ++ .../DateTimeFormatProcessorTests.java | 145 ++ .../xpack/sql/parser/SqlParserTests.java | 30 +- .../logical/command/sys/SysTablesTests.java | 77 +- .../sql/planner/QueryTranslatorTests.java | 16 + .../xpack/sql/type/SqlDataTypesTests.java | 6 +- .../xpack/test/rest/XPackRestIT.java | 18 +- .../api/async_search.delete.json | 3 +- .../rest-api-spec/api/async_search.get.json | 11 +- .../api/async_search.submit.json | 14 +- ...autoscaling.delete_autoscaling_policy.json | 24 + .../autoscaling.get_autoscaling_decision.json | 3 +- .../autoscaling.get_autoscaling_policy.json | 24 + .../autoscaling.put_autoscaling_policy.json | 28 + .../api/cat.ml_data_frame_analytics.json | 17 +- .../rest-api-spec/api/cat.ml_datafeeds.json | 17 +- .../rest-api-spec/api/cat.ml_jobs.json | 17 +- .../api/cat.ml_trained_models.json | 17 +- ...cat_transform.json => cat.transforms.json} | 19 +- .../api/ccr.delete_auto_follow_pattern.json | 3 +- .../rest-api-spec/api/ccr.follow.json | 3 +- .../rest-api-spec/api/ccr.follow_info.json | 3 +- .../rest-api-spec/api/ccr.follow_stats.json | 3 +- .../api/ccr.forget_follower.json | 3 +- .../api/ccr.get_auto_follow_pattern.json | 3 +- .../api/ccr.pause_auto_follow_pattern.json | 3 +- .../rest-api-spec/api/ccr.pause_follow.json | 3 +- .../api/ccr.put_auto_follow_pattern.json | 3 +- .../api/ccr.resume_auto_follow_pattern.json | 3 +- .../rest-api-spec/api/ccr.resume_follow.json | 3 +- .../rest-api-spec/api/ccr.stats.json | 3 +- .../rest-api-spec/api/ccr.unfollow.json | 3 +- ...transform_deprecated.delete_transform.json | 3 +- ...me_transform_deprecated.get_transform.json | 3 +- ...nsform_deprecated.get_transform_stats.json | 3 +- ...ransform_deprecated.preview_transform.json | 3 +- ...me_transform_deprecated.put_transform.json | 13 +- ..._transform_deprecated.start_transform.json | 3 +- ...e_transform_deprecated.stop_transform.json | 3 +- ...transform_deprecated.update_transform.json | 45 +- .../api/enrich.delete_policy.json | 3 +- .../api/enrich.execute_policy.json | 3 +- .../rest-api-spec/api/enrich.get_policy.json | 3 +- .../rest-api-spec/api/enrich.put_policy.json | 3 +- .../rest-api-spec/api/enrich.stats.json | 3 +- .../rest-api-spec/api/eql.search.json | 3 +- .../rest-api-spec/api/graph.explore.json | 3 +- .../api/ilm.delete_lifecycle.json | 3 +- .../api/ilm.explain_lifecycle.json | 3 +- .../rest-api-spec/api/ilm.get_lifecycle.json | 3 +- .../rest-api-spec/api/ilm.get_status.json | 3 +- .../rest-api-spec/api/ilm.move_to_step.json | 3 +- .../rest-api-spec/api/ilm.put_lifecycle.json | 3 +- .../rest-api-spec/api/ilm.remove_policy.json | 3 +- .../rest-api-spec/api/ilm.retry.json | 3 +- .../rest-api-spec/api/ilm.start.json | 3 +- .../resources/rest-api-spec/api/ilm.stop.json | 3 +- .../rest-api-spec/api/indices.freeze.json | 4 +- .../api/indices.reload_search_analyzers.json | 4 +- .../rest-api-spec/api/indices.unfreeze.json | 4 +- .../rest-api-spec/api/license.delete.json | 3 +- .../rest-api-spec/api/license.get.json | 7 +- .../api/license.get_basic_status.json | 3 +- .../api/license.get_trial_status.json | 3 +- .../rest-api-spec/api/license.post.json | 3 +- .../api/license.post_start_basic.json | 3 +- .../api/license.post_start_trial.json | 3 +- .../api/migration.deprecations.json | 3 +- .../rest-api-spec/api/ml.close_job.json | 3 +- .../rest-api-spec/api/ml.delete_calendar.json | 3 +- .../api/ml.delete_calendar_event.json | 3 +- .../api/ml.delete_calendar_job.json | 3 +- .../api/ml.delete_data_frame_analytics.json | 3 +- .../rest-api-spec/api/ml.delete_datafeed.json | 3 +- .../api/ml.delete_expired_data.json | 3 +- .../rest-api-spec/api/ml.delete_filter.json | 3 +- .../rest-api-spec/api/ml.delete_forecast.json | 5 +- .../rest-api-spec/api/ml.delete_job.json | 3 +- .../api/ml.delete_model_snapshot.json | 3 +- .../api/ml.delete_trained_model.json | 3 +- .../api/ml.estimate_model_memory.json | 3 +- .../api/ml.evaluate_data_frame.json | 3 +- .../api/ml.explain_data_frame_analytics.json | 35 +- .../api/ml.find_file_structure.json | 3 +- .../rest-api-spec/api/ml.flush_job.json | 3 +- .../rest-api-spec/api/ml.forecast.json | 3 +- .../rest-api-spec/api/ml.get_buckets.json | 3 +- .../api/ml.get_calendar_events.json | 3 +- .../rest-api-spec/api/ml.get_calendars.json | 3 +- .../rest-api-spec/api/ml.get_categories.json | 3 +- .../api/ml.get_data_frame_analytics.json | 3 +- .../ml.get_data_frame_analytics_stats.json | 3 +- .../api/ml.get_datafeed_stats.json | 3 +- .../rest-api-spec/api/ml.get_datafeeds.json | 3 +- .../rest-api-spec/api/ml.get_filters.json | 3 +- .../rest-api-spec/api/ml.get_influencers.json | 6 +- .../rest-api-spec/api/ml.get_job_stats.json | 3 +- .../rest-api-spec/api/ml.get_jobs.json | 3 +- .../api/ml.get_model_snapshots.json | 3 +- .../api/ml.get_overall_buckets.json | 3 +- .../rest-api-spec/api/ml.get_records.json | 9 +- .../api/ml.get_trained_models.json | 21 +- .../api/ml.get_trained_models_stats.json | 3 +- .../resources/rest-api-spec/api/ml.info.json | 3 +- .../rest-api-spec/api/ml.open_job.json | 3 +- .../api/ml.post_calendar_events.json | 3 +- .../rest-api-spec/api/ml.post_data.json | 3 +- .../api/ml.preview_datafeed.json | 3 +- .../rest-api-spec/api/ml.put_calendar.json | 3 +- .../api/ml.put_calendar_job.json | 3 +- .../api/ml.put_data_frame_analytics.json | 3 +- .../rest-api-spec/api/ml.put_datafeed.json | 5 +- .../rest-api-spec/api/ml.put_filter.json | 3 +- .../rest-api-spec/api/ml.put_job.json | 3 +- .../api/ml.put_trained_model.json | 5 +- .../api/ml.revert_model_snapshot.json | 3 +- .../api/ml.set_upgrade_mode.json | 3 +- .../api/ml.start_data_frame_analytics.json | 3 +- .../rest-api-spec/api/ml.start_datafeed.json | 3 +- .../api/ml.stop_data_frame_analytics.json | 3 +- .../rest-api-spec/api/ml.stop_datafeed.json | 3 +- .../rest-api-spec/api/ml.update_datafeed.json | 5 +- .../rest-api-spec/api/ml.update_filter.json | 3 +- .../rest-api-spec/api/ml.update_job.json | 3 +- .../api/ml.update_model_snapshot.json | 3 +- .../rest-api-spec/api/ml.validate.json | 3 +- .../api/ml.validate_detector.json | 3 +- .../rest-api-spec/api/monitoring.bulk.json | 3 +- .../rest-api-spec/api/rollup.delete_job.json | 3 +- .../rest-api-spec/api/rollup.get_jobs.json | 3 +- .../api/rollup.get_rollup_caps.json | 3 +- .../api/rollup.get_rollup_index_caps.json | 3 +- .../rest-api-spec/api/rollup.put_job.json | 3 +- .../api/rollup.rollup_search.json | 7 +- .../rest-api-spec/api/rollup.start_job.json | 3 +- .../rest-api-spec/api/rollup.stop_job.json | 3 +- .../api/searchable_snapshots.clear_cache.json | 55 + .../api/searchable_snapshots.mount.json | 43 + .../api/searchable_snapshots.stats.json | 30 + .../api/security.authenticate.json | 3 +- .../api/security.change_password.json | 3 +- .../api/security.clear_cached_realms.json | 3 +- .../api/security.clear_cached_roles.json | 3 +- .../api/security.create_api_key.json | 3 +- .../api/security.delete_privileges.json | 3 +- .../api/security.delete_role.json | 3 +- .../api/security.delete_role_mapping.json | 3 +- .../api/security.delete_user.json | 3 +- .../api/security.disable_user.json | 3 +- .../api/security.enable_user.json | 3 +- .../api/security.get_api_key.json | 7 +- .../api/security.get_builtin_privileges.json | 3 +- .../api/security.get_privileges.json | 3 +- .../rest-api-spec/api/security.get_role.json | 3 +- .../api/security.get_role_mapping.json | 3 +- .../rest-api-spec/api/security.get_token.json | 3 +- .../rest-api-spec/api/security.get_user.json | 3 +- .../api/security.get_user_privileges.json | 3 +- .../api/security.has_privileges.json | 3 +- .../api/security.invalidate_api_key.json | 3 +- .../api/security.invalidate_token.json | 3 +- .../api/security.put_privileges.json | 3 +- .../rest-api-spec/api/security.put_role.json | 3 +- .../api/security.put_role_mapping.json | 3 +- .../rest-api-spec/api/security.put_user.json | 3 +- .../api/slm.delete_lifecycle.json | 3 +- .../api/slm.execute_lifecycle.json | 3 +- .../api/slm.execute_retention.json | 3 +- .../rest-api-spec/api/slm.get_lifecycle.json | 3 +- .../rest-api-spec/api/slm.get_stats.json | 3 +- .../rest-api-spec/api/slm.get_status.json | 3 +- .../rest-api-spec/api/slm.put_lifecycle.json | 3 +- .../rest-api-spec/api/slm.start.json | 3 +- .../resources/rest-api-spec/api/slm.stop.json | 3 +- .../rest-api-spec/api/sql.query.json | 2 +- .../rest-api-spec/api/ssl.certificates.json | 3 +- .../api/transform.delete_transform.json | 3 +- .../api/transform.get_transform.json | 3 +- .../api/transform.get_transform_stats.json | 3 +- .../api/transform.preview_transform.json | 3 +- .../api/transform.put_transform.json | 13 +- .../api/transform.start_transform.json | 3 +- .../api/transform.stop_transform.json | 45 +- .../api/transform.update_transform.json | 45 +- .../rest-api-spec/api/watcher.ack_watch.json | 3 +- .../api/watcher.activate_watch.json | 3 +- .../api/watcher.deactivate_watch.json | 3 +- .../api/watcher.delete_watch.json | 3 +- .../api/watcher.execute_watch.json | 3 +- .../rest-api-spec/api/watcher.get_watch.json | 3 +- .../rest-api-spec/api/watcher.put_watch.json | 3 +- .../rest-api-spec/api/watcher.start.json | 3 +- .../rest-api-spec/api/watcher.stats.json | 3 +- .../rest-api-spec/api/watcher.stop.json | 3 +- .../rest-api-spec/api/xpack.info.json | 3 +- .../rest-api-spec/api/xpack.usage.json | 3 +- .../test/analytics/string_stats.yml | 53 + .../test/async_search/10_basic.yml | 10 +- .../test/ml/data_frame_analytics_crud.yml | 30 + .../test/ml/estimate_model_memory.yml | 187 +- .../test/ml/explain_data_frame_analytics.yml | 12 +- .../rest-api-spec/test/ml/filter_crud.yml | 22 - .../rest-api-spec/test/ml/inference_crud.yml | 54 + .../test/ml/inference_processor.yml | 40 + .../test/ml/inference_stats_crud.yml | 9 +- .../test/ml/ml_classic_analyze.yml | 4 +- .../test/ml/trained_model_cat_apis.yml | 2 + .../test/transform/transforms_cat_apis.yml | 12 +- .../transform/integration/TransformIT.java | 10 +- .../integration/TransformAuditorIT.java | 6 +- ...taDataIT.java => TransformMetadataIT.java} | 4 +- .../TransformPivotRestSpecialCasesIT.java | 111 + .../xpack/transform/Transform.java | 16 +- .../TransformClusterStateListener.java | 6 +- .../TransformUsageTransportAction.java | 8 +- .../transform/action/TransformNodes.java | 14 +- .../TransportDeleteTransformAction.java | 4 +- .../TransportGetTransformStatsAction.java | 8 +- .../TransportPreviewTransformAction.java | 4 +- .../action/TransportPutTransformAction.java | 8 +- .../action/TransportStartTransformAction.java | 36 +- .../action/TransportStopTransformAction.java | 30 +- .../TransportUpdateTransformAction.java | 8 +- .../transform/persistence/TransformIndex.java | 26 +- .../persistence/TransformInternalIndex.java | 50 +- .../rest/action/RestCatTransformAction.java | 44 +- .../transforms/TransformIndexer.java | 120 +- .../TransformPersistentTasksExecutor.java | 14 +- .../transform/transforms/TransformTask.java | 4 +- .../pivot/AggregationResultUtils.java | 9 +- .../transforms/pivot/Aggregations.java | 5 +- .../transform/transforms/pivot/Pivot.java | 47 +- .../xpack/transform/TransformTests.java | 19 +- .../transform/action/TransformNodesTests.java | 14 +- .../TransportStopTransformActionTests.java | 30 +- .../TransformInternalIndexTests.java | 18 +- ...TransformPersistentTasksExecutorTests.java | 73 +- .../pivot/AggregationResultUtilsTests.java | 5 + .../vectors/VectorsUsageTransportAction.java | 12 +- .../mapper/DenseVectorFieldMapperTests.java | 4 +- .../mapper/SparseVectorFieldMapperTests.java | 6 +- .../coordination/VotingOnlyNodePlugin.java | 2 +- x-pack/plugin/watcher/build.gradle | 4 +- .../elasticsearch/xpack/watcher/Watcher.java | 6 +- .../watcher/WatcherIndexingListener.java | 14 +- .../watcher/WatcherLifeCycleService.java | 14 +- .../xpack/watcher/WatcherService.java | 28 +- .../watcher/execution/ExecutionService.java | 2 +- .../execution/TriggeredWatchStore.java | 12 +- .../xpack/watcher/history/HistoryStore.java | 8 +- .../notification/email/HtmlSanitizer.java | 34 +- .../support/WatcherIndexTemplateRegistry.java | 8 +- .../TransportWatcherServiceAction.java | 20 +- .../stats/TransportWatcherStatsAction.java | 14 +- .../xpack/watcher/watch/WatchStoreUtils.java | 20 +- .../watcher/WatcherIndexingListenerTests.java | 78 +- .../WatcherInfoTransportActionTests.java | 4 +- .../watcher/WatcherLifeCycleServiceTests.java | 152 +- ...=> WatcherMetadataSerializationTests.java} | 56 +- .../xpack/watcher/WatcherServiceTests.java | 30 +- .../http/SizeLimitInputStreamTests.java | 4 +- .../CompareConditionSearchTests.java | 2 +- .../condition/ScriptConditionTests.java | 6 +- .../execution/TriggeredWatchStoreTests.java | 76 +- .../HistoryTemplateHttpMappingsTests.java | 6 +- .../HistoryTemplateTimeMappingsTests.java | 6 +- ...HistoryTemplateTransformMappingsTests.java | 2 +- .../support/WatcherDateTimeUtilsTests.java | 22 - .../WatcherIndexTemplateRegistryTests.java | 16 +- .../AbstractWatcherIntegrationTestCase.java | 4 +- .../test/integration/BootStrapTests.java | 6 +- .../test/integration/SingleNodeTests.java | 6 +- .../ack/TransportAckWatchActionTests.java | 6 +- .../TransportWatcherStatsActionTests.java | 4 +- .../wildcard/mapper/WildcardFieldMapper.java | 162 +- .../mapper/WildcardFieldMapperTests.java | 92 +- ...SpnegoHttpClientConfigCallbackHandler.java | 6 +- .../build.gradle | 4 +- .../build.gradle | 4 +- .../authc/oidc/OpenIdConnectAuthIT.java | 2 +- .../org/elasticsearch/test/OpenLdapTests.java | 18 +- x-pack/qa/rolling-upgrade-basic/build.gradle | 2 - .../upgrades/BasicLicenseUpgradeIT.java | 2 + .../upgrades/TransformSurvivesUpgradeIT.java | 12 +- .../UpgradeClusterClientYamlTestSuiteIT.java | 2 +- .../authc/saml/SamlAuthenticationIT.java | 8 +- x-pack/qa/security-tools-tests/build.gradle | 2 +- .../SmokeTestMonitoringWithSecurityIT.java | 1 + .../xpack/test/rest/XPackRestTestHelper.java | 2 + 3351 files changed, 94326 insertions(+), 38783 deletions(-) rename .eclipseformat.xml => buildSrc/formatterConfig.xml (99%) delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/checkstyle/SnippetLengthCheck.java rename buildSrc/src/main/{groovy/org/elasticsearch/gradle/test/Fixture.groovy => java/org/elasticsearch/gradle/test/Fixture.java} (89%) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/checkstyle/SnipptLengthCheckTests.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ComponentTemplatesExistRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DeleteComponentTemplateRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DeleteIndexTemplateV2Request.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetComponentTemplatesRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetComponentTemplatesResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplateV2Request.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesV2Response.java rename client/rest-high-level/src/main/java/org/elasticsearch/client/indices/{IndexTemplateMetaData.java => IndexTemplateMetadata.java} (76%) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateV2ExistRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutComponentTemplateRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateV2Request.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EstimateModelMemoryResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ClassificationConfig.java rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketAggregationBuilder.java => client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/InferenceConfig.java (73%) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/RegressionConfig.java rename client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/{WatcherMetaData.java => WatcherMetadata.java} (91%) create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/HighLevelRestClientCompressionIT.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/AsyncSearchDocumentationIT.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesV2ResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/ClassificationConfigTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/RegressionConfigTests.java create mode 100644 docs/java-rest/high-level/asyncsearch/delete.asciidoc create mode 100644 docs/java-rest/high-level/asyncsearch/get.asciidoc create mode 100644 docs/java-rest/high-level/asyncsearch/submit.asciidoc create mode 100644 docs/java-rest/high-level/ml/estimate-model-memory.asciidoc create mode 100644 docs/reference/aggregations/metrics/t-test-aggregation.asciidoc create mode 100644 docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc create mode 100644 docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc create mode 100644 docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc create mode 100644 docs/reference/eql/functions.asciidoc create mode 100644 docs/reference/mapping/types/point.asciidoc create mode 100644 docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc create mode 100644 docs/reference/searchable-snapshots/apis/clear-cache.asciidoc create mode 100644 docs/reference/searchable-snapshots/apis/get-stats.asciidoc create mode 100644 docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc create mode 100644 docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc create mode 100644 gradle/forbidden-dependencies.gradle create mode 100644 gradle/formatting.gradle create mode 100644 modules/geo/build.gradle create mode 100644 modules/geo/src/main/java/org/elasticsearch/geo/GeoPlugin.java create mode 100644 modules/geo/src/test/java/org/elasticsearch/geo/GeoClientYamlTestSuiteIT.java create mode 100644 modules/geo/src/test/java/org/elasticsearch/geo/GeoTests.java create mode 100644 modules/geo/src/test/resources/rest-api-spec/test/geo_shape/10_basic.yml create mode 100644 modules/kibana/build.gradle create mode 100644 modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java create mode 100644 modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java create mode 100644 modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java delete mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java delete mode 100644 modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufBytesReference.java delete mode 100644 modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java delete mode 100644 modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java create mode 100644 plugins/ingest-attachment/licenses/SparseBitSet-1.2.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/SparseBitSet-LICENSE.txt rename plugins/{repository-gcs/licenses/httpclient-NOTICE.txt => ingest-attachment/licenses/SparseBitSet-NOTICE.txt} (56%) delete mode 100644 plugins/ingest-attachment/licenses/commons-compress-1.18.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/commons-compress-1.19.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/commons-math3-3.6.1.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/commons-math3-LICENSE.txt create mode 100644 plugins/ingest-attachment/licenses/commons-math3-NOTICE.txt delete mode 100644 plugins/ingest-attachment/licenses/fontbox-2.0.16.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/fontbox-2.0.19.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/pdfbox-2.0.16.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/pdfbox-2.0.19.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/poi-4.0.1.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/poi-4.1.2.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-4.0.1.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-4.1.2.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.0.1.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-schemas-4.1.2.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/poi-scratchpad-4.0.1.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/poi-scratchpad-4.1.2.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/tika-core-1.22.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/tika-core-1.24.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/tika-parsers-1.22.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/tika-parsers-1.24.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/httpclient-4.5.10.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/httpclient-LICENSE.txt delete mode 100644 plugins/repository-gcs/licenses/httpcore-4.4.12.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-annotations-2.8.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-auth-2.8.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-client-2.8.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-common-2.8.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-common-2.8.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.1.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1 create mode 100644 qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/ESJsonLogsConfigIT.java create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/cluster.exists_component_template.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_index_template.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/20_get_missing.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml create mode 100644 rest-api-spec/src/main/resources/schema.json create mode 100644 server/licenses/ecs-logging-core-0.1.3.jar.sha1 create mode 100644 server/licenses/ecs-logging-core-LICENSE.txt create mode 100644 server/licenses/ecs-logging-core-NOTICE.txt create mode 100644 server/licenses/log4j2-ecs-layout-0.1.3.jar.sha1 create mode 100644 server/licenses/log4j2-ecs-layout-LICENSE.txt create mode 100644 server/licenses/log4j2-ecs-layout-NOTICE.txt create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateV2Action.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateV2Action.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplateV2Action.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplateV2Action.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2Action.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java rename server/src/main/java/org/elasticsearch/cluster/{MergableCustomMetaData.java => MergableCustomMetadata.java} (85%) rename server/src/main/java/org/elasticsearch/cluster/coordination/{CoordinationMetaData.java => CoordinationMetadata.java} (93%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{AliasMetaData.java => AliasMetadata.java} (85%) delete mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java rename server/src/main/java/org/elasticsearch/cluster/metadata/{IndexMetaData.java => IndexMetadata.java} (88%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{IndexTemplateMetaData.java => IndexTemplateMetadata.java} (84%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{MappingMetaData.java => MappingMetadata.java} (91%) delete mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaData.java => Metadata.java} (76%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaDataCreateIndexService.java => MetadataCreateIndexService.java} (67%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaDataDeleteIndexService.java => MetadataDeleteIndexService.java} (92%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaDataIndexAliasesService.java => MetadataIndexAliasesService.java} (87%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaDataIndexStateService.java => MetadataIndexStateService.java} (91%) create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaDataIndexUpgradeService.java => MetadataIndexUpgradeService.java} (81%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaDataMappingService.java => MetadataMappingService.java} (88%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaDataUpdateSettingsService.java => MetadataUpdateSettingsService.java} (80%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{RepositoriesMetaData.java => RepositoriesMetadata.java} (85%) rename server/src/main/java/org/elasticsearch/cluster/metadata/{RepositoryMetaData.java => RepositoryMetadata.java} (91%) create mode 100644 server/src/main/java/org/elasticsearch/cluster/routing/allocation/ExistingShardsAllocator.java rename server/src/main/java/org/elasticsearch/cluster/routing/allocation/{IndexMetaDataUpdater.java => IndexMetadataUpdater.java} (82%) rename server/src/main/java/org/elasticsearch/common/blobstore/{BlobMetaData.java => BlobMetadata.java} (97%) rename server/src/main/java/org/elasticsearch/common/blobstore/support/{PlainBlobMetaData.java => PlainBlobMetadata.java} (87%) create mode 100644 server/src/main/java/org/elasticsearch/common/logging/ClusterIdConverter.java create mode 100644 server/src/main/java/org/elasticsearch/common/logging/ECSJsonLayout.java create mode 100644 server/src/main/java/org/elasticsearch/common/logging/NodeIdConverter.java rename server/src/main/java/org/elasticsearch/env/{NodeMetaData.java => NodeMetadata.java} (83%) rename server/src/main/java/org/elasticsearch/gateway/{MetaDataStateFormat.java => MetadataStateFormat.java} (98%) create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/LowercaseNormalizer.java create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/LowercaseNormalizerProvider.java rename server/src/main/java/org/elasticsearch/index/shard/{ShardStateMetaData.java => ShardStateMetadata.java} (84%) rename server/src/main/java/org/elasticsearch/index/store/{StoreFileMetaData.java => StoreFileMetadata.java} (94%) rename server/src/main/java/org/elasticsearch/indices/store/{TransportNodesListShardStoreMetaData.java => TransportNodesListShardStoreMetadata.java} (79%) rename server/src/main/java/org/elasticsearch/persistent/{PersistentTasksCustomMetaData.java => PersistentTasksCustomMetadata.java} (93%) rename server/src/main/java/org/elasticsearch/plugins/{MetaDataUpgrader.java => MetadataUpgrader.java} (62%) create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateV2Action.java create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateV2Action.java create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateV2Action.java create mode 100644 server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java rename server/src/main/java/org/elasticsearch/script/{ScriptMetaData.java => ScriptMetadata.java} (87%) create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorProvider.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinMaxAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/AggregatorSupplier.java delete mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/package-info.java create mode 100644 server/src/main/java/org/elasticsearch/transport/Header.java create mode 100644 server/src/main/java/org/elasticsearch/transport/InboundAggregator.java create mode 100644 server/src/main/java/org/elasticsearch/transport/InboundDecoder.java create mode 100644 server/src/main/java/org/elasticsearch/transport/InboundPipeline.java create mode 100644 server/src/main/java/org/elasticsearch/transport/StatsTracker.java create mode 100644 server/src/main/java/org/elasticsearch/transport/TransportDecompressor.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateV2RequestTests.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplateV2RequestTests.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplateV2ResponseTests.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2RequestTests.java rename server/src/test/java/org/elasticsearch/cluster/coordination/{CoordinationMetaDataTests.java => CoordinationMetadataTests.java} (92%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{AliasMetaDataTests.java => AliasMetadataTests.java} (83%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{AliasOrIndexTests.java => IndexAbstractionTests.java} (75%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{IndexMetaDataTests.java => IndexMetadataTests.java} (69%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{IndexTemplateMetaDataTests.java => IndexTemplateMetadataTests.java} (86%) delete mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java rename server/src/test/java/org/elasticsearch/cluster/metadata/{MetaDataCreateIndexServiceTests.java => MetadataCreateIndexServiceTests.java} (85%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{MetaDataDeleteIndexServiceTests.java => MetadataDeleteIndexServiceTests.java} (91%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{MetaDataIndexAliasesServiceTests.java => MetadataIndexAliasesServiceTests.java} (79%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{MetaDataIndexStateServiceTests.java => MetadataIndexStateServiceTests.java} (81%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{MetaDataIndexStateServiceUtils.java => MetadataIndexStateServiceUtils.java} (83%) create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java rename server/src/test/java/org/elasticsearch/cluster/metadata/{MetaDataIndexUpgradeServiceTests.java => MetadataIndexUpgradeServiceTests.java} (63%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{MetaDataMappingServiceTests.java => MetadataMappingServiceTests.java} (88%) rename server/src/test/java/org/elasticsearch/cluster/metadata/{MetaDataTests.java => MetadataTests.java} (65%) delete mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java rename server/src/test/java/org/elasticsearch/env/{NodeMetaDataTests.java => NodeMetadataTests.java} (73%) rename server/src/test/java/org/elasticsearch/gateway/{MetaDataWriteDataNodesIT.java => MetadataNodesIT.java} (88%) rename server/src/test/java/org/elasticsearch/gateway/{MetaDataStateFormatTests.java => MetadataStateFormatTests.java} (96%) rename server/src/test/java/org/elasticsearch/persistent/{PersistentTasksCustomMetaDataTests.java => PersistentTasksCustomMetadataTests.java} (81%) rename server/src/test/java/org/elasticsearch/script/{ScriptMetaDataTests.java => ScriptMetadataTests.java} (85%) rename server/src/test/java/org/elasticsearch/search/aggregations/{MetaDataIT.java => MetadataIT.java} (79%) create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/BinaryTermsAggregatorTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/KeywordTermsAggregatorTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregatorTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilderTests.java rename server/src/test/java/org/elasticsearch/snapshots/{RepositoriesMetaDataSerializationTests.java => RepositoriesMetadataSerializationTests.java} (71%) create mode 100644 server/src/test/java/org/elasticsearch/transport/InboundAggregatorTests.java create mode 100644 server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java delete mode 100644 server/src/test/java/org/elasticsearch/transport/InboundMessageTests.java create mode 100644 server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java create mode 100644 server/src/test/java/org/elasticsearch/transport/TransportDecompressorTests.java create mode 100644 test/framework/src/main/java/org/elasticsearch/common/breaker/TestCircuitBreaker.java create mode 100644 test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java rename test/framework/src/main/java/org/elasticsearch/test/{TestCustomMetaData.java => TestCustomMetadata.java} (85%) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/TestGeoShapeFieldMapperPlugin.java rename server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeRequest.java => test/framework/src/main/java/org/elasticsearch/transport/TestRequest.java (63%) create mode 100644 test/framework/src/main/java/org/elasticsearch/transport/TestResponse.java create mode 100644 test/framework/src/main/java/org/elasticsearch/transport/TestTransportChannel.java create mode 100644 x-pack/plugin/analytics/licenses/commons-math3-3.2.jar.sha1 create mode 100644 x-pack/plugin/analytics/licenses/commons-math3-LICENSE.txt create mode 100644 x-pack/plugin/analytics/licenses/commons-math3-NOTICE.txt create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedHDRPercentilesAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AbstractHistoBackedTDigestPercentilesAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsPercentilesAggregatorFactory.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/AnalyticsValuesSourceType.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/support/HistogramValuesSource.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorSupplier.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorSupplier.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/InternalTTest.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/PairedTTestAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/PairedTTestState.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTest.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorFactory.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestState.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestStats.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestStatsBuilder.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestType.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/UnpairedTTestAggregator.java create mode 100644 x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/UnpairedTTestState.java create mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/InternalTTestTests.java create mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilderTests.java create mode 100644 x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java create mode 100644 x-pack/plugin/async-search/qa/rest/build.gradle create mode 100644 x-pack/plugin/async-search/qa/rest/src/main/java/org/elasticsearch/query/DeprecatedQueryBuilder.java create mode 100644 x-pack/plugin/async-search/qa/rest/src/main/java/org/elasticsearch/query/DeprecatedQueryPlugin.java create mode 100644 x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/qa/AsyncSearchRestIT.java create mode 100644 x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/async-search/10_deprecation.yml rename x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/{AsyncSearchActionTests.java => AsyncSearchActionIT.java} (97%) create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchActionTests.java create mode 100644 x-pack/plugin/autoscaling/qa/rest/src/test/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml create mode 100644 x-pack/plugin/autoscaling/qa/rest/src/test/resources/rest-api-spec/test/autoscaling/get_autoscaling_policy.yml create mode 100644 x-pack/plugin/autoscaling/qa/rest/src/test/resources/rest-api-spec/test/autoscaling/put_autoscaling_policy.yml create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/DeleteAutoscalingPolicyAction.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyAction.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/decision/AlwaysAutoscalingDecider.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/decision/AutoscalingDecider.java rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/{ => decision}/AutoscalingDecision.java (97%) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/{ => decision}/AutoscalingDecisionType.java (97%) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/{ => decision}/AutoscalingDecisions.java (92%) create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/policy/AutoscalingPolicy.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/policy/AutoscalingPolicyMetadata.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/rest/RestDeleteAutoscalingPolicyHandler.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/rest/RestGetAutoscalingPolicyHandler.java create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/rest/RestPutAutoscalingPolicyHandler.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingIntegTestCase.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadataDiffableSerializationTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/LocalStateAutoscaling.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/DeleteAutoscalingPolicyActionRequestWireSerializingTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionActionRequestWireSerializingTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingDecisionActionResponseWireSerializingTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionRequestWireSerializingTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionResponseWireSerializingTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyActionRequestWireSerializingTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionIT.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/{ => decision}/AutoscalingDecisionTests.java (90%) rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/{ => decision}/AutoscalingDecisionTypeWireSerializingTests.java (97%) rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/{ => decision}/AutoscalingDecisionWireSerializingTests.java (86%) rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/{ => decision}/AutoscalingDecisionsTests.java (92%) rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/{ => decision}/AutoscalingDecisionsWireSerializingTests.java (86%) create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/policy/AutoscalingPolicyMetadataDiffableSerializationTests.java create mode 100644 x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/policy/AutoscalingPolicySerializingTests.java create mode 100644 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutorAssignmentTests.java rename x-pack/plugin/core/src/main/java/org/elasticsearch/license/{LicensesMetaData.java => LicensesMetadata.java} (85%) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncActionBranchingStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationPersister.java rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/stats/{ => common}/MemoryUsage.java (83%) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfigUpdate.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStats.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LenientlyParsedInferenceConfig.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/StrictlyParsedInferenceConfig.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/{TokenMetaData.java => TokenMetadata.java} (90%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/ldap/support/{LdapMetaDataResolverSettings.java => LdapMetadataResolverSettings.java} (75%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/{WatcherMetaData.java => WatcherMetadata.java} (81%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/license/{LicensesMetaDataSerializationTests.java => LicensesMetadataSerializationTests.java} (61%) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AsyncActionBranchingStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CopySettingsStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SwapAliasesAndDeleteSourceIndexStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractBWCWireSerializationTestCase.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationPersisterTests.java rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/{ => common}/MemoryUsageTests.java (81%) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStatsTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStatsTests.java rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/{TokenMetaDataTests.java => TokenMetadataTests.java} (79%) create mode 100644 x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json create mode 100644 x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml create mode 100644 x-pack/plugin/eql/src/main/antlr/EqlBase.tokens create mode 100644 x-pack/plugin/eql/src/main/antlr/EqlBaseLexer.tokens create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Between.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Length.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContains.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContainsFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContainsFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Wildcard.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/KeyedFilter.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/StringUtils.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessorTests.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWithProcessorTests.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthProcessorTests.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithProcessorTests.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContainsFunctionProcessorTests.java create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/IdentityProviderAuthenticationIT.java create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/java/org/elasticsearch/xpack/idp/WildcardServiceProviderRestIT.java create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/idp-metadata.xml create mode 100644 x-pack/plugin/identity-provider/qa/idp-rest-tests/src/test/resources/wildcard_services.json create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/privileges/ApplicationActionsResolver.java create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderFactory.java create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/ServiceProviderCacheSettings.java create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProvider.java create mode 100644 x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolver.java rename x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/{TransportSamlInitiateSingleSignOnRequestTests.java => TransportSamlInitiateSingleSignOnActionTests.java} (78%) create mode 100644 x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java create mode 100644 x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/SecondaryAuthorizationUtils.java rename x-pack/plugin/ml/src/test/java/org/elasticsearch/license/{MachineLearningLicensingTests.java => MachineLearningLicensingIT.java} (97%) create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlFiltersIT.java create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/BaseSurrogateFunction.java create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/SurrogateFunction.java create mode 100644 x-pack/plugin/searchable-snapshots/build.gradle create mode 100644 x-pack/plugin/searchable-snapshots/qa/azure/build.gradle create mode 100644 x-pack/plugin/searchable-snapshots/qa/azure/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java create mode 100644 x-pack/plugin/searchable-snapshots/qa/build.gradle create mode 100644 x-pack/plugin/searchable-snapshots/qa/rest/build.gradle create mode 100644 x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/FsSearchableSnapshotsIT.java create mode 100644 x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/SearchableSnapshotsClientYamlTestSuiteIT.java create mode 100644 x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/clear_cache.yml create mode 100644 x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml create mode 100644 x-pack/plugin/searchable-snapshots/qa/s3/build.gradle create mode 100644 x-pack/plugin/searchable-snapshots/qa/s3/src/test/java/org/elasticsearch/xpack/searchablesnapshots/s3/S3SearchableSnapshotsIT.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/BaseSearchableSnapshotIndexInput.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/InMemoryNoOpCommitDirectory.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/IndexInputStats.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/CacheFile.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/CacheKey.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInput.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/SparseFileTracker.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/direct/DirectBlobContainerIndexInput.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotAllocator.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotIndexEventListener.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/AbstractTransportSearchableSnapshotsAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/ClearSearchableSnapshotsCacheAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/ClearSearchableSnapshotsCacheRequest.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/ClearSearchableSnapshotsCacheResponse.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsStatsAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsStatsRequest.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsStatsResponse.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/CacheService.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestClearSearchableSnapshotsCacheAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestMountSearchableSnapshotAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestSearchableSnapshotsStatsAction.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/InMemoryNoOpCommitDirectoryTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/IndexInputStatsTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CacheFileTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CacheKeyTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/CachedBlobContainerIndexInputTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/SparseFileTrackerTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/TestUtils.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/direct/DirectBlobContainerIndexInputTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java create mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsStatsResponseTests.java rename x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/{LdapMetaDataResolver.java => LdapMetadataResolver.java} (91%) rename x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/{LdapMetaDataResolverTests.java => LdapMetadataResolverTests.java} (89%) create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianPoint.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryPointProcessor.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverPointTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverShapeTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverPointTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java rename x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/{DatabaseMetadataProxy.java => DatabaseMetaDataProxy.java} (82%) create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BinaryDateTimeDatePartFunction.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormat.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormatPipe.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormatProcessor.java create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormatPipeTests.java create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormatProcessorTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/autoscaling.delete_autoscaling_policy.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/autoscaling.get_autoscaling_policy.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/autoscaling.put_autoscaling_policy.json rename x-pack/plugin/src/test/resources/rest-api-spec/api/{transform.cat_transform.json => cat.transforms.json} (88%) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/searchable_snapshots.clear_cache.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/searchable_snapshots.mount.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/searchable_snapshots.stats.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/string_stats.yml rename x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/{TransformMetaDataIT.java => TransformMetadataIT.java} (95%) rename x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/{WatcherMetaDataSerializationTests.java => WatcherMetadataSerializationTests.java} (57%) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index caaf385149c48..0243adb1a6226 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -17,5 +17,7 @@ BWC_VERSION: - "7.6.0" - "7.6.1" - "7.6.2" + - "7.6.3" - "7.7.0" + - "7.8.0" - "8.0.0" diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties index 46f7eab9cde60..f632332b8d789 100644 --- a/.ci/java-versions.properties +++ b/.ci/java-versions.properties @@ -4,6 +4,6 @@ # build and test Elasticsearch for this branch. Valid Java versions # are 'java' or 'openjdk' followed by the major release number. -ES_BUILD_JAVA=openjdk13 +ES_BUILD_JAVA=openjdk14 ES_RUNTIME_JAVA=openjdk11 GRADLE_TASK=build diff --git a/.ci/jobs.t/defaults.yml b/.ci/jobs.t/defaults.yml index f1a774538c1a3..5f95c89a80d72 100644 --- a/.ci/jobs.t/defaults.yml +++ b/.ci/jobs.t/defaults.yml @@ -49,6 +49,7 @@ JAVA11_HOME=$HOME/.java/java11 JAVA12_HOME=$HOME/.java/openjdk12 JAVA13_HOME=$HOME/.java/openjdk13 + JAVA14_HOME=$HOME/.java/openjdk14 GRADLE_OPTS=-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m properties: - github: diff --git a/.ci/os.ps1 b/.ci/os.ps1 index 10b9c1bd8f6e1..54f5ca54ffaa3 100644 --- a/.ci/os.ps1 +++ b/.ci/os.ps1 @@ -5,7 +5,7 @@ If (-NOT ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdent exit } -$AppProps = ConvertFrom-StringData (Get-Content .ci/java-versions.properties -raw) +$AppProps = ConvertFrom-StringData (Get-Content .ci/java-versions.properties -raw) $env:ES_BUILD_JAVA=$AppProps.ES_BUILD_JAVA $env:ES_RUNTIME_JAVA=$AppProps.ES_RUNTIME_JAVA @@ -26,6 +26,6 @@ New-Item -ItemType directory -Path \tmp $ErrorActionPreference="Continue" # TODO: remove the task exclusions once dependencies are set correctly and these don't run for Windows or buldiung the deb on windows is fixed -& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --scan --console=plain destructiveDistroTest +& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain destructiveDistroTest exit $LastExitCode diff --git a/.ci/os.sh b/.ci/os.sh index 7cb94ab9fa93f..82490e09b4247 100755 --- a/.ci/os.sh +++ b/.ci/os.sh @@ -1,6 +1,6 @@ #!/bin/bash -# opensuse 15 has a missing dep for systemd +# opensuse 15 has a missing dep for systemd if which zypper > /dev/null ; then sudo zypper install -y insserv-compat @@ -31,21 +31,21 @@ cp -v .ci/init.gradle $HOME/.gradle/init.d unset JAVA_HOME -if ! [ -e "/usr/bin/bats" ] ; then +if ! [ -e "/usr/bin/bats" ] ; then git clone https://github.com/sstephenson/bats /tmp/bats sudo /tmp/bats/install.sh /usr fi -if [ -f "/etc/os-release" ] ; then +if [ -f "/etc/os-release" ] ; then cat /etc/os-release . /etc/os-release - if [[ "$ID" == "debian" || "$ID_LIKE" == "debian" ]] ; then + if [[ "$ID" == "debian" || "$ID_LIKE" == "debian" ]] ; then # FIXME: The base image should not have rpm installed sudo rm -Rf /usr/bin/rpm - # Work around incorrect lintian version - # https://github.com/elastic/elasticsearch/issues/48573 - if [ $VERSION_ID == 10 ] ; then + # Work around incorrect lintian version + # https://github.com/elastic/elasticsearch/issues/48573 + if [ $VERSION_ID == 10 ] ; then sudo apt-get install -y --allow-downgrades lintian=2.15.0 fi fi @@ -71,7 +71,7 @@ sudo chmod 0440 /etc/sudoers.d/elasticsearch_vars sudo rm -Rf /elasticsearch sudo mkdir -p /elasticsearch/qa/ && sudo chown jenkins /elasticsearch/qa/ && ln -s $PWD/qa/vagrant /elasticsearch/qa/ -# sudo sets it's own PATH thus we use env to override that and call sudo annother time so we keep the secure root PATH +# sudo sets it's own PATH thus we use env to override that and call sudo annother time so we keep the secure root PATH # run with --continue to run both bats and java tests even if one fails # be explicit about Gradle home dir so we use the same even with sudo sudo -E env \ @@ -79,5 +79,5 @@ sudo -E env \ RUNTIME_JAVA_HOME=`readlink -f -n $RUNTIME_JAVA_HOME` \ --unset=JAVA_HOME \ SYSTEM_JAVA_HOME=`readlink -f -n $RUNTIME_JAVA_HOME` \ - ./gradlew -g $HOME/.gradle --scan --parallel $@ --continue destructivePackagingTest + ./gradlew -g $HOME/.gradle --scan --parallel --continue $@ diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 1eb4f9576c4c6..3cd4abcc21090 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -22,5 +22,6 @@ export JAVA8_HOME="${HOME}"/.java/java8 export JAVA11_HOME="${HOME}"/.java/java11 export JAVA12_HOME="${HOME}"/.java/openjdk12 export JAVA13_HOME="${HOME}"/.java/openjdk13 +export JAVA14_HOME="${HOME}"/.java/openjdk14 ./gradlew --parallel clean -s resolveAllDependencies diff --git a/.idea/runConfigurations/Debug_Elasticsearch.xml b/.idea/runConfigurations/Debug_Elasticsearch.xml index 685c59c1846a4..6b58e18d22266 100644 --- a/.idea/runConfigurations/Debug_Elasticsearch.xml +++ b/.idea/runConfigurations/Debug_Elasticsearch.xml @@ -5,7 +5,11 @@