From b0942e9a5bd517865197de1021e35d72ae654024 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Sep 2019 20:27:53 -0600 Subject: [PATCH 01/12] WIP --- .../transport/ProxyConnectionStrategy.java | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java new file mode 100644 index 0000000000000..48b12cef9d017 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.AlreadyClosedException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ContextPreservingActionListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +public class ProxyConnectionStrategy implements TransportConnectionListener { + + private static final Logger logger = LogManager.getLogger(ProxyConnectionStrategy.class); + + private static final int MAX_LISTENERS = 100; + private final AtomicBoolean closed = new AtomicBoolean(false); + private final Object mutex = new Object(); + private final ConnectionManager connectionManager; + private final ThreadPool threadPool; + private final int maxNumRemoteConnections; + private List> listeners = new ArrayList<>(); + + ProxyConnectionStrategy(ConnectionManager connectionManager, ThreadPool threadPool, int maxNumRemoteConnections) { + this.connectionManager = connectionManager; + this.threadPool = threadPool; + this.maxNumRemoteConnections = maxNumRemoteConnections; + } + + /** + * Triggers a connect round unless there is one running already. If there is a connect round running, the listener will either + * be queued or rejected and failed. + */ + void connect(ActionListener connectListener) { + boolean runConnect = false; + final ActionListener listener = + ContextPreservingActionListener.wrapPreservingContext(connectListener, threadPool.getThreadContext()); + boolean closed; + synchronized (mutex) { + closed = this.closed.get(); + if (closed) { + assert listeners.isEmpty(); + } else { + if (listeners.size() >= MAX_LISTENERS) { + assert listeners.size() == MAX_LISTENERS; + listener.onFailure(new RejectedExecutionException("connect queue is full")); + return; + } else { + listeners.add(listener); + } + runConnect = listeners.size() == 1; + } + } + if (closed) { + connectListener.onFailure(new AlreadyClosedException("connect handler is already closed")); + return; + } + if (runConnect) { + ExecutorService executor = threadPool.executor(ThreadPool.Names.MANAGEMENT); +// executor.submit(new AbstractRunnable() { +// @Override +// public void onFailure(Exception e) { +// ActionListener.onFailure(getAndClearListeners(), e); +// } +// +// @Override +// protected void doRun() { +// collectRemoteNodes(seedNodes.stream().map(Tuple::v2).iterator(), +// new ActionListener<>() { +// @Override +// public void onResponse(Void aVoid) { +// ActionListener.onResponse(getAndClearListeners(), aVoid); +// } +// +// @Override +// public void onFailure(Exception e) { +// ActionListener.onFailure(getAndClearListeners(), e); +// } +// }); +// } +// }); + } + } + + @Override + public void onNodeDisconnected(DiscoveryNode node) { + if (connectionManager.size() < maxNumRemoteConnections) { + // try to reconnect and fill up the slot of the disconnected node + connect(ActionListener.wrap( + ignore -> logger.trace("successfully connected after disconnect of {}", node), + e -> logger.trace(() -> new ParameterizedMessage("failed to connect after disconnect of {}", node), e))); + } + } + + private List> getAndClearListeners() { + final List> result; + synchronized (mutex) { + if (listeners.isEmpty()) { + result = Collections.emptyList(); + } else { + result = listeners; + listeners = new ArrayList<>(); + } + } + return result; + } +} From d6393ce6734ae6cbe23166b5033f71d95b0e3dec Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Sep 2019 15:55:55 -0600 Subject: [PATCH 02/12] WIP --- .../transport/ProxyConnectionStrategy.java | 320 +++++++++++++----- .../transport/RemoteConnectionStrategy.java | 139 ++++++++ 2 files changed, 375 insertions(+), 84 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java index 48b12cef9d017..3353fd7dc8672 100644 --- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java @@ -19,115 +19,267 @@ package org.elasticsearch.transport; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.ContextPreservingActionListener; +import org.elasticsearch.action.StepListener; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.threadpool.ThreadPool; -import java.util.ArrayList; +import java.io.IOException; +import java.net.InetSocketAddress; import java.util.Collections; +import java.util.Iterator; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.function.Supplier; -public class ProxyConnectionStrategy implements TransportConnectionListener { +import static org.elasticsearch.common.settings.Setting.intSetting; - private static final Logger logger = LogManager.getLogger(ProxyConnectionStrategy.class); +public class ProxyConnectionStrategy extends RemoteConnectionStrategy { - private static final int MAX_LISTENERS = 100; - private final AtomicBoolean closed = new AtomicBoolean(false); - private final Object mutex = new Object(); + /** + * Whether the connection to the remote cluster is through a proxy. + */ + public static final Setting REMOTE_CONNECTIONS_PER_CLUSTER = Setting.boolSetting( + "cluster.remote.proxy_mode", false, Setting.Property.NodeScope); + + /** + * The maximum number of connections that will be established to the proxy in front of the remote cluster. + */ + public static final Setting.AffixSetting REMOTE_CLUSTER_PROXY_CONNECTIONS = Setting.affixKeySetting( + "cluster.remote.", + "proxy_connections_per_cluster", + key -> intSetting(key, 15, 1, Setting.Property.NodeScope)); + + private final String clusterAlias; + private final TransportService transportService; private final ConnectionManager connectionManager; - private final ThreadPool threadPool; - private final int maxNumRemoteConnections; - private List> listeners = new ArrayList<>(); + private final Predicate nodePredicate; + private final SetOnce remoteClusterName = new SetOnce<>(); + private volatile String proxyAddress; - ProxyConnectionStrategy(ConnectionManager connectionManager, ThreadPool threadPool, int maxNumRemoteConnections) { + ProxyConnectionStrategy(String clusterAlias, TransportService transportService, ConnectionManager connectionManager, + ThreadPool threadPool, String proxyAddress, int maxNumRemoteConnections, + Predicate nodePredicate) { + super(threadPool, connectionManager, maxNumRemoteConnections); + this.clusterAlias = clusterAlias; + this.transportService = transportService; this.connectionManager = connectionManager; - this.threadPool = threadPool; - this.maxNumRemoteConnections = maxNumRemoteConnections; + this.proxyAddress = proxyAddress; + this.nodePredicate = nodePredicate; } - /** - * Triggers a connect round unless there is one running already. If there is a connect round running, the listener will either - * be queued or rejected and failed. - */ - void connect(ActionListener connectListener) { - boolean runConnect = false; - final ActionListener listener = - ContextPreservingActionListener.wrapPreservingContext(connectListener, threadPool.getThreadContext()); - boolean closed; - synchronized (mutex) { - closed = this.closed.get(); - if (closed) { - assert listeners.isEmpty(); - } else { - if (listeners.size() >= MAX_LISTENERS) { - assert listeners.size() == MAX_LISTENERS; - listener.onFailure(new RejectedExecutionException("connect queue is full")); - return; - } else { - listeners.add(listener); + @Override + protected void connectImpl(ActionListener listener) { + final List>> seedNodes = Collections.emptyList(); + collectRemoteNodes(seedNodes.stream().map(Tuple::v2).iterator(), listener); + } + + private void collectRemoteNodes(Iterator> seedNodes, ActionListener listener) { + if (Thread.currentThread().isInterrupted()) { + listener.onFailure(new InterruptedException("remote connect thread got interrupted")); + } + + if (seedNodes.hasNext()) { + final Consumer onFailure = e -> { + if (e instanceof ConnectTransportException || + e instanceof IOException || + e instanceof IllegalStateException) { + // ISE if we fail the handshake with an version incompatible node + if (seedNodes.hasNext()) { + logger.debug(() -> new ParameterizedMessage( + "fetching nodes from external cluster [{}] failed moving to next node", clusterAlias), e); + collectRemoteNodes(seedNodes, listener); + return; + } } - runConnect = listeners.size() == 1; + logger.warn(() -> new ParameterizedMessage("fetching nodes from external cluster [{}] failed", clusterAlias), e); + listener.onFailure(e); + }; + + final DiscoveryNode seedNode = maybeAddProxyAddress(proxyAddress, seedNodes.next().get()); + logger.debug("[{}] opening connection to seed node: [{}] proxy address: [{}]", clusterAlias, seedNode, + proxyAddress); + final ConnectionProfile profile = ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG); + final StepListener openConnectionStep = new StepListener<>(); + try { + connectionManager.openConnection(seedNode, profile, openConnectionStep); + } catch (Exception e) { + onFailure.accept(e); } + + final StepListener handShakeStep = new StepListener<>(); + openConnectionStep.whenComplete(connection -> { + ConnectionProfile connectionProfile = connectionManager.getConnectionProfile(); + transportService.handshake(connection, connectionProfile.getHandshakeTimeout().millis(), + getRemoteClusterNamePredicate(), handShakeStep); + }, onFailure); + + final StepListener fullConnectionStep = new StepListener<>(); + handShakeStep.whenComplete(handshakeResponse -> { + final DiscoveryNode handshakeNode = maybeAddProxyAddress(proxyAddress, handshakeResponse.getDiscoveryNode()); + + if (nodePredicate.test(handshakeNode) && shouldOpenMoreConnections()) { + connectionManager.connectToNode(handshakeNode, null, + transportService.connectionValidator(handshakeNode), fullConnectionStep); + } else { + fullConnectionStep.onResponse(null); + } + }, e -> { + final Transport.Connection connection = openConnectionStep.result(); + logger.warn(new ParameterizedMessage("failed to connect to seed node [{}]", connection.getNode()), e); + IOUtils.closeWhileHandlingException(connection); + onFailure.accept(e); + }); + + fullConnectionStep.whenComplete(aVoid -> { + if (remoteClusterName.get() == null) { + TransportService.HandshakeResponse handshakeResponse = handShakeStep.result(); + assert handshakeResponse.getClusterName().value() != null; + remoteClusterName.set(handshakeResponse.getClusterName()); + } + final Transport.Connection connection = openConnectionStep.result(); + + ClusterStateRequest request = new ClusterStateRequest(); + request.clear(); + request.nodes(true); + // here we pass on the connection since we can only close it once the sendRequest returns otherwise + // due to the async nature (it will return before it's actually sent) this can cause the request to fail + // due to an already closed connection. + ThreadPool threadPool = transportService.getThreadPool(); + ThreadContext threadContext = threadPool.getThreadContext(); + TransportService.ContextRestoreResponseHandler responseHandler = new TransportService + .ContextRestoreResponseHandler<>(threadContext.newRestorableContext(false), + new SniffClusterStateResponseHandler(connection, listener, seedNodes)); + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { + // we stash any context here since this is an internal execution and should not leak any + // existing context information. + threadContext.markAsSystemContext(); + transportService.sendRequest(connection, ClusterStateAction.NAME, request, TransportRequestOptions.EMPTY, + responseHandler); + } + }, e -> { + IOUtils.closeWhileHandlingException(openConnectionStep.result()); + onFailure.accept(e); + }); + } else { + listener.onFailure(new IllegalStateException("no seed node left")); } - if (closed) { - connectListener.onFailure(new AlreadyClosedException("connect handler is already closed")); - return; + } + + /* This class handles the _state response from the remote cluster when sniffing nodes to connect to */ + private class SniffClusterStateResponseHandler implements TransportResponseHandler { + + private final Transport.Connection connection; + private final ActionListener listener; + private final Iterator> seedNodes; + + SniffClusterStateResponseHandler(Transport.Connection connection, ActionListener listener, + Iterator> seedNodes) { + this.connection = connection; + this.listener = listener; + this.seedNodes = seedNodes; } - if (runConnect) { - ExecutorService executor = threadPool.executor(ThreadPool.Names.MANAGEMENT); -// executor.submit(new AbstractRunnable() { -// @Override -// public void onFailure(Exception e) { -// ActionListener.onFailure(getAndClearListeners(), e); -// } -// -// @Override -// protected void doRun() { -// collectRemoteNodes(seedNodes.stream().map(Tuple::v2).iterator(), -// new ActionListener<>() { -// @Override -// public void onResponse(Void aVoid) { -// ActionListener.onResponse(getAndClearListeners(), aVoid); -// } -// -// @Override -// public void onFailure(Exception e) { -// ActionListener.onFailure(getAndClearListeners(), e); -// } -// }); -// } -// }); + + @Override + public ClusterStateResponse read(StreamInput in) throws IOException { + return new ClusterStateResponse(in); } - } - @Override - public void onNodeDisconnected(DiscoveryNode node) { - if (connectionManager.size() < maxNumRemoteConnections) { - // try to reconnect and fill up the slot of the disconnected node - connect(ActionListener.wrap( - ignore -> logger.trace("successfully connected after disconnect of {}", node), - e -> logger.trace(() -> new ParameterizedMessage("failed to connect after disconnect of {}", node), e))); + @Override + public void handleResponse(ClusterStateResponse response) { + handleNodes(response.getState().nodes().getNodes().valuesIt()); + } + + private void handleNodes(Iterator nodesIter) { + while (nodesIter.hasNext()) { + final DiscoveryNode node = maybeAddProxyAddress(proxyAddress, nodesIter.next()); + if (nodePredicate.test(node) && shouldOpenMoreConnections()) { + connectionManager.connectToNode(node, null, + transportService.connectionValidator(node), new ActionListener<>() { + @Override + public void onResponse(Void aVoid) { + handleNodes(nodesIter); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof ConnectTransportException || + e instanceof IllegalStateException) { + // ISE if we fail the handshake with an version incompatible node + // fair enough we can't connect just move on + logger.debug(() -> new ParameterizedMessage("failed to connect to node {}", node), e); + handleNodes(nodesIter); + } else { + logger.warn(() -> + new ParameterizedMessage("fetching nodes from external cluster {} failed", clusterAlias), e); + IOUtils.closeWhileHandlingException(connection); + collectRemoteNodes(seedNodes, listener); + } + } + }); + return; + } + } + // We have to close this connection before we notify listeners - this is mainly needed for test correctness + // since if we do it afterwards we might fail assertions that check if all high level connections are closed. + // from a code correctness perspective we could also close it afterwards. + IOUtils.closeWhileHandlingException(connection); + listener.onResponse(null); + } + + @Override + public void handleException(TransportException exp) { + logger.warn(() -> new ParameterizedMessage("fetching nodes from external cluster {} failed", clusterAlias), exp); + try { + IOUtils.closeWhileHandlingException(connection); + } finally { + // once the connection is closed lets try the next node + collectRemoteNodes(seedNodes, listener); + } + } + + @Override + public String executor() { + return ThreadPool.Names.MANAGEMENT; } } - private List> getAndClearListeners() { - final List> result; - synchronized (mutex) { - if (listeners.isEmpty()) { - result = Collections.emptyList(); - } else { - result = listeners; - listeners = new ArrayList<>(); + private Predicate getRemoteClusterNamePredicate() { + return new Predicate<>() { + @Override + public boolean test(ClusterName c) { + return remoteClusterName.get() == null || c.equals(remoteClusterName.get()); + } + + @Override + public String toString() { + return remoteClusterName.get() == null ? "any cluster name" + : "expected remote cluster name [" + remoteClusterName.get().value() + "]"; } + }; + } + + private static DiscoveryNode maybeAddProxyAddress(String proxyAddress, DiscoveryNode node) { + if (proxyAddress == null || proxyAddress.isEmpty()) { + return node; + } else { + // resolve proxy address lazy here + InetSocketAddress proxyInetAddress = RemoteClusterAware.parseSeedAddress(proxyAddress); + return new DiscoveryNode(node.getName(), node.getId(), node.getEphemeralId(), node.getHostName(), node + .getHostAddress(), new TransportAddress(proxyInetAddress), node.getAttributes(), node.getRoles(), node.getVersion()); } - return result; } } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java new file mode 100644 index 0000000000000..c3123adde1dac --- /dev/null +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -0,0 +1,139 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.AlreadyClosedException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ContextPreservingActionListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class RemoteConnectionStrategy implements TransportConnectionListener { + + protected static final Logger logger = LogManager.getLogger(RemoteConnectionStrategy.class); + + private static final int MAX_LISTENERS = 100; + private final AtomicBoolean closed = new AtomicBoolean(false); + private final Object mutex = new Object(); + private final ThreadPool threadPool; + private final ConnectionManager connectionManager; + private final int maxNumRemoteConnections; + private List> listeners = new ArrayList<>(); + + RemoteConnectionStrategy(ThreadPool threadPool, ConnectionManager connectionManager, int maxNumRemoteConnections) { + this.threadPool = threadPool; + this.connectionManager = connectionManager; + this.maxNumRemoteConnections = maxNumRemoteConnections; + } + + /** + * Triggers a connect round unless there is one running already. If there is a connect round running, the listener will either + * be queued or rejected and failed. + */ + void connect(ActionListener connectListener) { + boolean runConnect = false; + final ActionListener listener = + ContextPreservingActionListener.wrapPreservingContext(connectListener, threadPool.getThreadContext()); + boolean closed; + synchronized (mutex) { + closed = this.closed.get(); + if (closed) { + assert listeners.isEmpty(); + } else { + if (listeners.size() >= MAX_LISTENERS) { + assert listeners.size() == MAX_LISTENERS; + listener.onFailure(new RejectedExecutionException("connect listener queue is full")); + return; + } else { + listeners.add(listener); + } + runConnect = listeners.size() == 1; + } + } + if (closed) { + connectListener.onFailure(new AlreadyClosedException("connect handler is already closed")); + return; + } + if (runConnect) { + ExecutorService executor = threadPool.executor(ThreadPool.Names.MANAGEMENT); + executor.submit(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + ActionListener.onFailure(getAndClearListeners(), e); + } + + @Override + protected void doRun() { + connectImpl(new ActionListener<>() { + @Override + public void onResponse(Void aVoid) { + ActionListener.onResponse(getAndClearListeners(), aVoid); + } + + @Override + public void onFailure(Exception e) { + ActionListener.onFailure(getAndClearListeners(), e); + } + }); + } + }); + } + } + + @Override + public void onNodeDisconnected(DiscoveryNode node) { + if (shouldOpenMoreConnections()) { + // try to reconnect and fill up the slot of the disconnected node + connect(ActionListener.wrap( + ignore -> logger.trace("successfully connected after disconnect of {}", node), + e -> logger.trace(() -> new ParameterizedMessage("failed to connect after disconnect of {}", node), e))); + } + } + + protected boolean shouldOpenMoreConnections() { + return connectionManager.size() < maxNumRemoteConnections; + } + + protected abstract void connectImpl(ActionListener listener); + + private List> getAndClearListeners() { + final List> result; + synchronized (mutex) { + if (listeners.isEmpty()) { + result = Collections.emptyList(); + } else { + result = listeners; + listeners = new ArrayList<>(); + } + } + return result; + } +} From 63a7c0bd54817997501c62257f3cc68e41863021 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Sep 2019 22:30:53 -0600 Subject: [PATCH 03/12] Changes --- ...tionStrategy.java => DirectConnectionStrategy.java} | 10 ++++------ .../transport/RemoteConnectionStrategy.java | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) rename server/src/main/java/org/elasticsearch/transport/{ProxyConnectionStrategy.java => DirectConnectionStrategy.java} (96%) diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/DirectConnectionStrategy.java similarity index 96% rename from server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java rename to server/src/main/java/org/elasticsearch/transport/DirectConnectionStrategy.java index 3353fd7dc8672..065b2ce7a86fd 100644 --- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/DirectConnectionStrategy.java @@ -47,7 +47,7 @@ import static org.elasticsearch.common.settings.Setting.intSetting; -public class ProxyConnectionStrategy extends RemoteConnectionStrategy { +public class DirectConnectionStrategy extends RemoteConnectionStrategy { /** * Whether the connection to the remote cluster is through a proxy. @@ -65,18 +65,16 @@ public class ProxyConnectionStrategy extends RemoteConnectionStrategy { private final String clusterAlias; private final TransportService transportService; - private final ConnectionManager connectionManager; private final Predicate nodePredicate; private final SetOnce remoteClusterName = new SetOnce<>(); private volatile String proxyAddress; - ProxyConnectionStrategy(String clusterAlias, TransportService transportService, ConnectionManager connectionManager, - ThreadPool threadPool, String proxyAddress, int maxNumRemoteConnections, - Predicate nodePredicate) { + DirectConnectionStrategy(String clusterAlias, TransportService transportService, ConnectionManager connectionManager, + ThreadPool threadPool, String proxyAddress, int maxNumRemoteConnections, + Predicate nodePredicate) { super(threadPool, connectionManager, maxNumRemoteConnections); this.clusterAlias = clusterAlias; this.transportService = transportService; - this.connectionManager = connectionManager; this.proxyAddress = proxyAddress; this.nodePredicate = nodePredicate; } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index c3123adde1dac..86bb3aa038a38 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -44,7 +44,7 @@ public abstract class RemoteConnectionStrategy implements TransportConnectionLis private final AtomicBoolean closed = new AtomicBoolean(false); private final Object mutex = new Object(); private final ThreadPool threadPool; - private final ConnectionManager connectionManager; + protected final ConnectionManager connectionManager; private final int maxNumRemoteConnections; private List> listeners = new ArrayList<>(); From e61b3cadcf5a8f31ffeafd6013a8ef9d03556f20 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 25 Sep 2019 17:23:21 -0600 Subject: [PATCH 04/12] WIP --- .../transport/RemoteClusterConnection.java | 366 +----------------- .../transport/RemoteClusterService.java | 4 +- .../transport/RemoteConnectionStrategy.java | 41 +- ...tegy.java => SniffConnectionStrategy.java} | 36 +- .../SniffConnectionStrategyTests.java | 23 ++ 5 files changed, 93 insertions(+), 377 deletions(-) rename server/src/main/java/org/elasticsearch/transport/{DirectConnectionStrategy.java => SniffConnectionStrategy.java} (91%) create mode 100644 server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 979d3bc6a1c23..c2d8802080642 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -18,41 +18,25 @@ */ package org.elasticsearch.transport; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.StepListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.support.ContextPreservingActionListener; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -73,20 +57,16 @@ */ final class RemoteClusterConnection implements TransportConnectionListener, Closeable { - private static final Logger logger = LogManager.getLogger(RemoteClusterConnection.class); - private final TransportService transportService; private final RemoteConnectionManager remoteConnectionManager; + private final RemoteConnectionStrategy connectionStrategy; private final String clusterAlias; private final int maxNumRemoteConnections; - private final Predicate nodePredicate; private final ThreadPool threadPool; - private volatile String proxyAddress; private volatile List>> seedNodes; + private volatile String proxyAddress; private volatile boolean skipUnavailable; - private final ConnectHandler connectHandler; private final TimeValue initialConnectionTimeout; - private final SetOnce remoteClusterName = new SetOnce<>(); /** * Creates a new {@link RemoteClusterConnection} @@ -112,33 +92,21 @@ final class RemoteClusterConnection implements TransportConnectionListener, Clos String proxyAddress, ConnectionManager connectionManager) { this.transportService = transportService; this.maxNumRemoteConnections = maxNumRemoteConnections; - this.nodePredicate = nodePredicate; this.clusterAlias = clusterAlias; this.remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + this.connectionStrategy = new SniffConnectionStrategy(clusterAlias, transportService, remoteConnectionManager, + transportService.getThreadPool(), proxyAddress, maxNumRemoteConnections, nodePredicate, + Collections.unmodifiableList(seedNodes)); + connectionManager.addListener(this.connectionStrategy); + // we register the transport service here as a listener to make sure we notify handlers on disconnect etc. + connectionManager.addListener(transportService); this.seedNodes = Collections.unmodifiableList(seedNodes); this.skipUnavailable = RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE .getConcreteSettingForNamespace(clusterAlias).get(settings); - this.connectHandler = new ConnectHandler(); this.threadPool = transportService.threadPool; - connectionManager.addListener(this); - // we register the transport service here as a listener to make sure we notify handlers on disconnect etc. - connectionManager.addListener(transportService); - this.proxyAddress = proxyAddress; initialConnectionTimeout = RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings); } - - private static DiscoveryNode maybeAddProxyAddress(String proxyAddress, DiscoveryNode node) { - if (proxyAddress == null || proxyAddress.isEmpty()) { - return node; - } else { - // resolve proxy address lazy here - InetSocketAddress proxyInetAddress = RemoteClusterAware.parseSeedAddress(proxyAddress); - return new DiscoveryNode(node.getName(), node.getId(), node.getEphemeralId(), node.getHostName(), node - .getHostAddress(), new TransportAddress(proxyInetAddress), node.getAttributes(), node.getRoles(), node.getVersion()); - } - } - /** * Updates the list of seed nodes for this cluster connection */ @@ -148,7 +116,8 @@ synchronized void updateSeedNodes( final ActionListener connectListener) { this.seedNodes = List.copyOf(seedNodes); this.proxyAddress = proxyAddress; - connectHandler.connect(connectListener); + // TODO: Update seedNodes + connectionStrategy.connect(connectListener); } /** @@ -165,23 +134,13 @@ boolean isSkipUnavailable() { return skipUnavailable; } - @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { - if (remoteConnectionManager.size() < maxNumRemoteConnections) { - // try to reconnect and fill up the slot of the disconnected node - connectHandler.connect(ActionListener.wrap( - ignore -> logger.trace("successfully connected after disconnect of {}", node), - e -> logger.trace(() -> new ParameterizedMessage("failed to connect after disconnect of {}", node), e))); - } - } - /** * Ensures that this cluster is connected. If the cluster is connected this operation * will invoke the listener immediately. */ void ensureConnected(ActionListener voidActionListener) { if (remoteConnectionManager.size() == 0) { - connectHandler.connect(voidActionListener); + connectionStrategy.connect(voidActionListener); } else { voidActionListener.onResponse(null); } @@ -256,316 +215,31 @@ Transport.Connection getConnection(DiscoveryNode remoteClusterNode) { return remoteConnectionManager.getRemoteConnection(remoteClusterNode); } - private Predicate getRemoteClusterNamePredicate() { - return - new Predicate() { - @Override - public boolean test(ClusterName c) { - return remoteClusterName.get() == null || c.equals(remoteClusterName.get()); - } - - @Override - public String toString() { - return remoteClusterName.get() == null ? "any cluster name" - : "expected remote cluster name [" + remoteClusterName.get().value() + "]"; - } - }; - } - Transport.Connection getConnection() { return remoteConnectionManager.getAnyRemoteConnection(); } @Override public void close() throws IOException { - IOUtils.close(connectHandler, remoteConnectionManager); + remoteConnectionManager.getConnectionManager().removeListener(connectionStrategy); + IOUtils.close(connectionStrategy, remoteConnectionManager); } public boolean isClosed() { - return connectHandler.isClosed(); + return connectionStrategy.isClosed(); } - public List>> getSeedNodes() { + List>> getSeedNodes() { return seedNodes; } - /** - * The connect handler manages node discovery and the actual connect to the remote cluster. - * There is at most one connect job running at any time. If such a connect job is triggered - * while another job is running the provided listeners are queued and batched up until the current running job returns. - * - * The handler has a built-in queue that can hold up to 100 connect attempts and will reject requests once the queue is full. - * In a scenario when a remote cluster becomes unavailable we will queue requests up but if we can't connect quick enough - * we will just reject the connect trigger which will lead to failing searches. - */ - private class ConnectHandler implements Closeable { - private static final int MAX_LISTENERS = 100; - private final AtomicBoolean closed = new AtomicBoolean(false); - private final Object mutex = new Object(); - private List> listeners = new ArrayList<>(); - - /** - * Triggers a connect round unless there is one running already. If there is a connect round running, the listener will either - * be queued or rejected and failed. - */ - void connect(ActionListener connectListener) { - boolean runConnect = false; - final ActionListener listener = - ContextPreservingActionListener.wrapPreservingContext(connectListener, threadPool.getThreadContext()); - boolean closed; - synchronized (mutex) { - closed = this.closed.get(); - if (closed) { - assert listeners.isEmpty(); - } else { - if (listeners.size() >= MAX_LISTENERS) { - assert listeners.size() == MAX_LISTENERS; - listener.onFailure(new RejectedExecutionException("connect queue is full")); - return; - } else { - listeners.add(listener); - } - runConnect = listeners.size() == 1; - } - } - if (closed) { - connectListener.onFailure(new AlreadyClosedException("connect handler is already closed")); - return; - } - if (runConnect) { - ExecutorService executor = threadPool.executor(ThreadPool.Names.MANAGEMENT); - executor.submit(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - ActionListener.onFailure(getAndClearListeners(), e); - } - - @Override - protected void doRun() { - collectRemoteNodes(seedNodes.stream().map(Tuple::v2).iterator(), - new ActionListener<>() { - @Override - public void onResponse(Void aVoid) { - ActionListener.onResponse(getAndClearListeners(), aVoid); - } - - @Override - public void onFailure(Exception e) { - ActionListener.onFailure(getAndClearListeners(), e); - } - }); - } - }); - } - } - - private List> getAndClearListeners() { - final List> result; - synchronized (mutex) { - if (listeners.isEmpty()) { - result = Collections.emptyList(); - } else { - result = listeners; - listeners = new ArrayList<>(); - } - } - return result; - } - - private void collectRemoteNodes(Iterator> seedNodes, ActionListener listener) { - if (Thread.currentThread().isInterrupted()) { - listener.onFailure(new InterruptedException("remote connect thread got interrupted")); - } - - if (seedNodes.hasNext()) { - final Consumer onFailure = e -> { - if (e instanceof ConnectTransportException || - e instanceof IOException || - e instanceof IllegalStateException) { - // ISE if we fail the handshake with an version incompatible node - if (seedNodes.hasNext()) { - logger.debug(() -> new ParameterizedMessage( - "fetching nodes from external cluster [{}] failed moving to next node", clusterAlias), e); - collectRemoteNodes(seedNodes, listener); - return; - } - } - logger.warn(() -> new ParameterizedMessage("fetching nodes from external cluster [{}] failed", clusterAlias), e); - listener.onFailure(e); - }; - - final DiscoveryNode seedNode = maybeAddProxyAddress(proxyAddress, seedNodes.next().get()); - logger.debug("[{}] opening connection to seed node: [{}] proxy address: [{}]", clusterAlias, seedNode, - proxyAddress); - final ConnectionProfile profile = ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG); - final StepListener openConnectionStep = new StepListener<>(); - try { - remoteConnectionManager.openConnection(seedNode, profile, openConnectionStep); - } catch (Exception e) { - onFailure.accept(e); - } - - final StepListener handShakeStep = new StepListener<>(); - openConnectionStep.whenComplete(connection -> { - ConnectionProfile connectionProfile = remoteConnectionManager.getConnectionManager().getConnectionProfile(); - transportService.handshake(connection, connectionProfile.getHandshakeTimeout().millis(), - getRemoteClusterNamePredicate(), handShakeStep); - }, onFailure); - - final StepListener fullConnectionStep = new StepListener<>(); - handShakeStep.whenComplete(handshakeResponse -> { - final DiscoveryNode handshakeNode = maybeAddProxyAddress(proxyAddress, handshakeResponse.getDiscoveryNode()); - - if (nodePredicate.test(handshakeNode) && remoteConnectionManager.size() < maxNumRemoteConnections) { - remoteConnectionManager.connectToNode(handshakeNode, null, - transportService.connectionValidator(handshakeNode), fullConnectionStep); - } else { - fullConnectionStep.onResponse(null); - } - }, e -> { - final Transport.Connection connection = openConnectionStep.result(); - logger.warn(new ParameterizedMessage("failed to connect to seed node [{}]", connection.getNode()), e); - IOUtils.closeWhileHandlingException(connection); - onFailure.accept(e); - }); - - fullConnectionStep.whenComplete(aVoid -> { - if (remoteClusterName.get() == null) { - TransportService.HandshakeResponse handshakeResponse = handShakeStep.result(); - assert handshakeResponse.getClusterName().value() != null; - remoteClusterName.set(handshakeResponse.getClusterName()); - } - final Transport.Connection connection = openConnectionStep.result(); - - ClusterStateRequest request = new ClusterStateRequest(); - request.clear(); - request.nodes(true); - // here we pass on the connection since we can only close it once the sendRequest returns otherwise - // due to the async nature (it will return before it's actually sent) this can cause the request to fail - // due to an already closed connection. - ThreadPool threadPool = transportService.getThreadPool(); - ThreadContext threadContext = threadPool.getThreadContext(); - TransportService.ContextRestoreResponseHandler responseHandler = new TransportService - .ContextRestoreResponseHandler<>(threadContext.newRestorableContext(false), - new SniffClusterStateResponseHandler(connection, listener, seedNodes)); - try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { - // we stash any context here since this is an internal execution and should not leak any - // existing context information. - threadContext.markAsSystemContext(); - transportService.sendRequest(connection, ClusterStateAction.NAME, request, TransportRequestOptions.EMPTY, - responseHandler); - } - }, e -> { - IOUtils.closeWhileHandlingException(openConnectionStep.result()); - onFailure.accept(e); - }); - } else { - listener.onFailure(new IllegalStateException("no seed node left")); - } - } - - @Override - public void close() throws IOException { - final List> toNotify; - synchronized (mutex) { - if (closed.compareAndSet(false, true)) { - toNotify = listeners; - listeners = Collections.emptyList(); - } else { - toNotify = Collections.emptyList(); - } - } - ActionListener.onFailure(toNotify, new AlreadyClosedException("connect handler is already closed")); - } - - final boolean isClosed() { - return closed.get(); - } - - /* This class handles the _state response from the remote cluster when sniffing nodes to connect to */ - private class SniffClusterStateResponseHandler implements TransportResponseHandler { - - private final Transport.Connection connection; - private final ActionListener listener; - private final Iterator> seedNodes; - - SniffClusterStateResponseHandler(Transport.Connection connection, ActionListener listener, - Iterator> seedNodes) { - this.connection = connection; - this.listener = listener; - this.seedNodes = seedNodes; - } - - @Override - public ClusterStateResponse read(StreamInput in) throws IOException { - return new ClusterStateResponse(in); - } - - @Override - public void handleResponse(ClusterStateResponse response) { - handleNodes(response.getState().nodes().getNodes().valuesIt()); - } - - private void handleNodes(Iterator nodesIter) { - while (nodesIter.hasNext()) { - final DiscoveryNode node = maybeAddProxyAddress(proxyAddress, nodesIter.next()); - if (nodePredicate.test(node) && remoteConnectionManager.size() < maxNumRemoteConnections) { - remoteConnectionManager.connectToNode(node, null, - transportService.connectionValidator(node), new ActionListener<>() { - @Override - public void onResponse(Void aVoid) { - handleNodes(nodesIter); - } - - @Override - public void onFailure(Exception e) { - if (e instanceof ConnectTransportException || - e instanceof IllegalStateException) { - // ISE if we fail the handshake with an version incompatible node - // fair enough we can't connect just move on - logger.debug(() -> new ParameterizedMessage("failed to connect to node {}", node), e); - handleNodes(nodesIter); - } else { - logger.warn(() -> - new ParameterizedMessage("fetching nodes from external cluster {} failed", clusterAlias), e); - IOUtils.closeWhileHandlingException(connection); - collectRemoteNodes(seedNodes, listener); - } - } - }); - return; - } - } - // We have to close this connection before we notify listeners - this is mainly needed for test correctness - // since if we do it afterwards we might fail assertions that check if all high level connections are closed. - // from a code correctness perspective we could also close it afterwards. - IOUtils.closeWhileHandlingException(connection); - listener.onResponse(null); - } - - @Override - public void handleException(TransportException exp) { - logger.warn(() -> new ParameterizedMessage("fetching nodes from external cluster {} failed", clusterAlias), exp); - try { - IOUtils.closeWhileHandlingException(connection); - } finally { - // once the connection is closed lets try the next node - collectRemoteNodes(seedNodes, listener); - } - } - - @Override - public String executor() { - return ThreadPool.Names.MANAGEMENT; - } - } + String getProxyAddress() { + return proxyAddress; } - boolean assertNoRunningConnections() { // for testing only - synchronized (connectHandler.mutex) { - assert connectHandler.listeners.isEmpty(); - } - return true; + // for testing only + boolean assertNoRunningConnections() { + return connectionStrategy.assertNoRunningConnections(); } boolean isNodeConnected(final DiscoveryNode node) { diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index db4e0f021e05b..3dff3c7921abf 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -184,7 +184,7 @@ private synchronized void updateRemoteClusters(Map { if (countDown.countDown()) { connectionListener.onResponse(response); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index 86bb3aa038a38..7b55ca84b616e 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -26,9 +26,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.threadpool.ThreadPool; +import java.io.Closeable; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -36,7 +39,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; -public abstract class RemoteConnectionStrategy implements TransportConnectionListener { +public abstract class RemoteConnectionStrategy implements TransportConnectionListener, Closeable { protected static final Logger logger = LogManager.getLogger(RemoteConnectionStrategy.class); @@ -44,14 +47,12 @@ public abstract class RemoteConnectionStrategy implements TransportConnectionLis private final AtomicBoolean closed = new AtomicBoolean(false); private final Object mutex = new Object(); private final ThreadPool threadPool; - protected final ConnectionManager connectionManager; - private final int maxNumRemoteConnections; + protected final RemoteConnectionManager connectionManager; private List> listeners = new ArrayList<>(); - RemoteConnectionStrategy(ThreadPool threadPool, ConnectionManager connectionManager, int maxNumRemoteConnections) { + RemoteConnectionStrategy(ThreadPool threadPool, RemoteConnectionManager connectionManager) { this.threadPool = threadPool; this.connectionManager = connectionManager; - this.maxNumRemoteConnections = maxNumRemoteConnections; } /** @@ -109,7 +110,7 @@ public void onFailure(Exception e) { } @Override - public void onNodeDisconnected(DiscoveryNode node) { + public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { if (shouldOpenMoreConnections()) { // try to reconnect and fill up the slot of the disconnected node connect(ActionListener.wrap( @@ -118,10 +119,34 @@ public void onNodeDisconnected(DiscoveryNode node) { } } - protected boolean shouldOpenMoreConnections() { - return connectionManager.size() < maxNumRemoteConnections; + @Override + public void close() throws IOException { + final List> toNotify; + synchronized (mutex) { + if (closed.compareAndSet(false, true)) { + toNotify = listeners; + listeners = Collections.emptyList(); + } else { + toNotify = Collections.emptyList(); + } + } + ActionListener.onFailure(toNotify, new AlreadyClosedException("connect handler is already closed")); + } + + public boolean isClosed() { + return closed.get(); + } + + // for testing only + boolean assertNoRunningConnections() { + synchronized (mutex) { + assert listeners.isEmpty(); + } + return true; } + protected abstract boolean shouldOpenMoreConnections(); + protected abstract void connectImpl(ActionListener listener); private List> getAndClearListeners() { diff --git a/server/src/main/java/org/elasticsearch/transport/DirectConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java similarity index 91% rename from server/src/main/java/org/elasticsearch/transport/DirectConnectionStrategy.java rename to server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java index 065b2ce7a86fd..cff8b07f19167 100644 --- a/server/src/main/java/org/elasticsearch/transport/DirectConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java @@ -47,44 +47,38 @@ import static org.elasticsearch.common.settings.Setting.intSetting; -public class DirectConnectionStrategy extends RemoteConnectionStrategy { - - /** - * Whether the connection to the remote cluster is through a proxy. - */ - public static final Setting REMOTE_CONNECTIONS_PER_CLUSTER = Setting.boolSetting( - "cluster.remote.proxy_mode", false, Setting.Property.NodeScope); - - /** - * The maximum number of connections that will be established to the proxy in front of the remote cluster. - */ - public static final Setting.AffixSetting REMOTE_CLUSTER_PROXY_CONNECTIONS = Setting.affixKeySetting( - "cluster.remote.", - "proxy_connections_per_cluster", - key -> intSetting(key, 15, 1, Setting.Property.NodeScope)); +public class SniffConnectionStrategy extends RemoteConnectionStrategy { private final String clusterAlias; + private final List>> seedNodes; private final TransportService transportService; + private final int maxNumRemoteConnections; private final Predicate nodePredicate; private final SetOnce remoteClusterName = new SetOnce<>(); private volatile String proxyAddress; - DirectConnectionStrategy(String clusterAlias, TransportService transportService, ConnectionManager connectionManager, - ThreadPool threadPool, String proxyAddress, int maxNumRemoteConnections, - Predicate nodePredicate) { - super(threadPool, connectionManager, maxNumRemoteConnections); + SniffConnectionStrategy(String clusterAlias, TransportService transportService, RemoteConnectionManager connectionManager, + ThreadPool threadPool, String proxyAddress, int maxNumRemoteConnections, + Predicate nodePredicate, List>> seedNodes) { + super(threadPool, connectionManager); this.clusterAlias = clusterAlias; this.transportService = transportService; this.proxyAddress = proxyAddress; + this.maxNumRemoteConnections = maxNumRemoteConnections; this.nodePredicate = nodePredicate; + this.seedNodes = seedNodes; } @Override protected void connectImpl(ActionListener listener) { - final List>> seedNodes = Collections.emptyList(); collectRemoteNodes(seedNodes.stream().map(Tuple::v2).iterator(), listener); } + @Override + protected boolean shouldOpenMoreConnections() { + return connectionManager.size() < maxNumRemoteConnections; + } + private void collectRemoteNodes(Iterator> seedNodes, ActionListener listener) { if (Thread.currentThread().isInterrupted()) { listener.onFailure(new InterruptedException("remote connect thread got interrupted")); @@ -120,7 +114,7 @@ private void collectRemoteNodes(Iterator> seedNodes, Act final StepListener handShakeStep = new StepListener<>(); openConnectionStep.whenComplete(connection -> { - ConnectionProfile connectionProfile = connectionManager.getConnectionProfile(); + ConnectionProfile connectionProfile = connectionManager.getConnectionManager().getConnectionProfile(); transportService.handshake(connection, connectionProfile.getHandshakeTimeout().millis(), getRemoteClusterNamePredicate(), handShakeStep); }, onFailure); diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java new file mode 100644 index 0000000000000..661cd3231096e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +public class SniffConnectionStrategyTests { +} From a82e098ba4b28f116504f10158aaec9529b74ad1 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 26 Sep 2019 12:44:13 -0600 Subject: [PATCH 05/12] WIP --- .../transport/RemoteClusterConnection.java | 2 +- .../transport/SniffConnectionStrategy.java | 6 ++-- .../SniffConnectionStrategyTests.java | 30 ++++++++++++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index c2d8802080642..ccd6b770dae91 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -95,7 +95,7 @@ final class RemoteClusterConnection implements TransportConnectionListener, Clos this.clusterAlias = clusterAlias; this.remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); this.connectionStrategy = new SniffConnectionStrategy(clusterAlias, transportService, remoteConnectionManager, - transportService.getThreadPool(), proxyAddress, maxNumRemoteConnections, nodePredicate, + proxyAddress, maxNumRemoteConnections, nodePredicate, Collections.unmodifiableList(seedNodes)); connectionManager.addListener(this.connectionStrategy); // we register the transport service here as a listener to make sure we notify handlers on disconnect etc. diff --git a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java index cff8b07f19167..9d6c381d83bf6 100644 --- a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java @@ -58,9 +58,9 @@ public class SniffConnectionStrategy extends RemoteConnectionStrategy { private volatile String proxyAddress; SniffConnectionStrategy(String clusterAlias, TransportService transportService, RemoteConnectionManager connectionManager, - ThreadPool threadPool, String proxyAddress, int maxNumRemoteConnections, - Predicate nodePredicate, List>> seedNodes) { - super(threadPool, connectionManager); + String proxyAddress, int maxNumRemoteConnections, Predicate nodePredicate, + List>> seedNodes) { + super(transportService.getThreadPool(), connectionManager); this.clusterAlias = clusterAlias; this.transportService = transportService; this.proxyAddress = proxyAddress; diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index 661cd3231096e..de780608217b8 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -19,5 +19,33 @@ package org.elasticsearch.transport; -public class SniffConnectionStrategyTests { +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.concurrent.TimeUnit; + +public class SniffConnectionStrategyTests extends ESTestCase { + + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Override + public void tearDown() throws Exception { + super.tearDown(); + ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); + } + + public void testThing() { + try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + service.start(); + service.acceptIncomingRequests(); + + SniffConnectionStrategy strategy = new SniffConnectionStrategy("cluster", service, null, null, 6, null, null); + } + + } + } From 60ccefbfefd379be6dcf9ad1bd7e4ea7716c6db5 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 26 Sep 2019 16:15:20 -0600 Subject: [PATCH 06/12] Changes --- .../transport/RemoteConnectionStrategy.java | 3 +- .../RemoteClusterConnectionTests.java | 593 +++++++++--------- .../SniffConnectionStrategyTests.java | 232 ++++++- .../test/transport/MockTransportService.java | 4 + 4 files changed, 530 insertions(+), 302 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index 7b55ca84b616e..abd7ac31228d7 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -53,6 +53,7 @@ public abstract class RemoteConnectionStrategy implements TransportConnectionLis RemoteConnectionStrategy(ThreadPool threadPool, RemoteConnectionManager connectionManager) { this.threadPool = threadPool; this.connectionManager = connectionManager; + connectionManager.getConnectionManager().addListener(this); } /** @@ -120,7 +121,7 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti } @Override - public void close() throws IOException { + public void close() { final List> toNotify; synchronized (mutex) { if (closed.compareAndSet(false, true)) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 92eb60cfdaf67..5bb2aa22a9dd1 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -150,7 +150,7 @@ public static MockTransportService startTransport( } SearchHits searchHits; if ("null_target".equals(request.preference())) { - searchHits = new SearchHits(new SearchHit[] {new SearchHit(0)}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1F); + searchHits = new SearchHits(new SearchHit[]{new SearchHit(0)}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1F); } else { searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); } @@ -180,234 +180,235 @@ public static MockTransportService startTransport( } } - public void testRemoteProfileIsUsedForLocalCluster() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, - profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, seedNodes(seedNode)); - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connection.assertNoRunningConnections()); - TransportFuture futureHandler = transportFuture(ClusterSearchShardsResponse::new); - TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.BULK) - .build(); - IllegalStateException ise = (IllegalStateException) expectThrows(SendRequestTransportException.class, () -> { - service.sendRequest(connectionManager.getConnection(discoverableNode), - ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), options, futureHandler); - futureHandler.txGet(); - }).getCause(); - assertEquals(ise.getMessage(), "can't select channel size is 0 for types: [RECOVERY, BULK, STATE]"); - } - } - } - } +// public void testRemoteProfileIsUsedForLocalCluster() throws Exception { +// List knownNodes = new CopyOnWriteArrayList<>(); +// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); +// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { +// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); +// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); +// knownNodes.add(seedTransport.getLocalDiscoNode()); +// knownNodes.add(discoverableTransport.getLocalDiscoNode()); +// Collections.shuffle(knownNodes, random()); +// +// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { +// service.start(); +// service.acceptIncomingRequests(); +// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", +// Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, +// profile)) { +// ConnectionManager connectionManager = connection.getConnectionManager(); +// updateSeedNodes(connection, seedNodes(seedNode)); +// assertTrue(connectionManager.nodeConnected(seedNode)); +// assertTrue(connectionManager.nodeConnected(discoverableNode)); +// assertTrue(connection.assertNoRunningConnections()); +// TransportFuture futureHandler = transportFuture(ClusterSearchShardsResponse::new); +// TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.BULK) +// .build(); +// IllegalStateException ise = (IllegalStateException) expectThrows(SendRequestTransportException.class, () -> { +// service.sendRequest(connectionManager.getConnection(discoverableNode), +// ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), options, futureHandler); +// futureHandler.txGet(); +// }).getCause(); +// assertEquals(ise.getMessage(), "can't select channel size is 0 for types: [RECOVERY, BULK, STATE]"); +// } +// } +// } +// } +// +// public void testRemoteProfileIsUsedForRemoteCluster() throws Exception { +// List knownNodes = new CopyOnWriteArrayList<>(); +// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT, threadPool, +// Settings.builder().put("cluster.name", "foobar").build()); +// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, +// threadPool, Settings.builder().put("cluster.name", "foobar").build())) { +// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); +// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); +// knownNodes.add(seedTransport.getLocalDiscoNode()); +// knownNodes.add(discoverableTransport.getLocalDiscoNode()); +// Collections.shuffle(knownNodes, random()); +// +// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { +// service.start(); +// service.acceptIncomingRequests(); +// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", +// Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, +// profile)) { +// ConnectionManager connectionManager = connection.getConnectionManager(); +// updateSeedNodes(connection, seedNodes(seedNode)); +// assertTrue(connectionManager.nodeConnected(seedNode)); +// assertTrue(connectionManager.nodeConnected(discoverableNode)); +// assertTrue(connection.assertNoRunningConnections()); +// TransportFuture futureHandler = transportFuture(ClusterSearchShardsResponse::new); +// TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.BULK) +// .build(); +// IllegalStateException ise = (IllegalStateException) expectThrows(SendRequestTransportException.class, () -> { +// service.sendRequest(connectionManager.getConnection(discoverableNode), +// ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), options, futureHandler); +// futureHandler.txGet(); +// }).getCause(); +// assertEquals(ise.getMessage(), "can't select channel size is 0 for types: [RECOVERY, BULK, STATE]"); +// +// TransportFuture handler = transportFuture(ClusterSearchShardsResponse::new); +// TransportRequestOptions ops = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.REG) +// .build(); +// service.sendRequest(connection.getConnection(), ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), +// ops, handler); +// handler.txGet(); +// } +// } +// } +// } +// +// public void testDiscoverSingleNode() throws Exception { +// List knownNodes = new CopyOnWriteArrayList<>(); +// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); +// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { +// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); +// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); +// knownNodes.add(seedTransport.getLocalDiscoNode()); +// knownNodes.add(discoverableTransport.getLocalDiscoNode()); +// Collections.shuffle(knownNodes, random()); +// +// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { +// service.start(); +// service.acceptIncomingRequests(); +// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", +// Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, +// profile)) { +// ConnectionManager connectionManager = connection.getConnectionManager(); +// updateSeedNodes(connection, seedNodes(seedNode)); +// assertTrue(connectionManager.nodeConnected(seedNode)); +// assertTrue(connectionManager.nodeConnected(discoverableNode)); +// assertTrue(connection.assertNoRunningConnections()); +// } +// } +// } +// } + +// public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception { +// List knownNodes = new CopyOnWriteArrayList<>(); +// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); +// MockTransportService incompatibleTransport = startTransport("incompat_seed_node", knownNodes, Version.fromString("2.0.0")); +// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { +// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); +// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); +// DiscoveryNode incompatibleSeedNode = incompatibleTransport.getLocalDiscoNode(); +// knownNodes.add(seedTransport.getLocalDiscoNode()); +// knownNodes.add(discoverableTransport.getLocalDiscoNode()); +// knownNodes.add(incompatibleTransport.getLocalDiscoNode()); +// Collections.shuffle(knownNodes, random()); +// List>> seedNodes = Arrays.asList( +// Tuple.tuple(incompatibleSeedNode.toString(), () -> incompatibleSeedNode), +// Tuple.tuple(seedNode.toString(), () -> seedNode)); +// Collections.shuffle(seedNodes, random()); +// +// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { +// service.start(); +// service.acceptIncomingRequests(); +// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", +// seedNodes, service, Integer.MAX_VALUE, n -> true, null, profile)) { +// ConnectionManager connectionManager = connection.getConnectionManager(); +// updateSeedNodes(connection, seedNodes); +// assertTrue(connectionManager.nodeConnected(seedNode)); +// assertTrue(connectionManager.nodeConnected(discoverableNode)); +// assertFalse(connectionManager.nodeConnected(incompatibleSeedNode)); +// assertTrue(connection.assertNoRunningConnections()); +// } +// } +// } +// } + +// public void testNodeDisconnected() throws Exception { +// List knownNodes = new CopyOnWriteArrayList<>(); +// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); +// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT); +// MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) { +// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); +// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); +// DiscoveryNode spareNode = spareTransport.getLocalDiscoNode(); +// knownNodes.add(seedTransport.getLocalDiscoNode()); +// knownNodes.add(discoverableTransport.getLocalDiscoNode()); +// Collections.shuffle(knownNodes, random()); +// +// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { +// service.start(); +// service.acceptIncomingRequests(); +// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", +// seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, profile)) { +// ConnectionManager connectionManager = connection.getConnectionManager(); +// updateSeedNodes(connection, seedNodes(seedNode)); +// assertTrue(connectionManager.nodeConnected(seedNode)); +// assertTrue(connectionManager.nodeConnected(discoverableNode)); +// assertFalse(connectionManager.nodeConnected(spareNode)); +// knownNodes.add(spareNode); +// CountDownLatch latchDisconnect = new CountDownLatch(1); +// CountDownLatch latchConnected = new CountDownLatch(1); +// connectionManager.addListener(new TransportConnectionListener() { +// @Override +// public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { +// if (node.equals(discoverableNode)) { +// latchDisconnect.countDown(); +// } +// } +// +// @Override +// public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) { +// if (node.equals(spareNode)) { +// latchConnected.countDown(); +// } +// } +// }); +// +// discoverableTransport.close(); +// // now make sure we try to connect again to other nodes once we got disconnected +// assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS)); +// assertTrue(latchConnected.await(10, TimeUnit.SECONDS)); +// assertTrue(connectionManager.nodeConnected(spareNode)); +// } +// } +// } +// } + +// public void testFilterDiscoveredNodes() throws Exception { +// List knownNodes = new CopyOnWriteArrayList<>(); +// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); +// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { +// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); +// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); +// knownNodes.add(seedTransport.getLocalDiscoNode()); +// knownNodes.add(discoverableTransport.getLocalDiscoNode()); +// DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode; +// Collections.shuffle(knownNodes, random()); +// +// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { +// service.start(); +// service.acceptIncomingRequests(); +// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", +// seedNodes(seedNode), service, Integer.MAX_VALUE, n -> n.equals(rejectedNode) == false, null, profile)) { +// ConnectionManager connectionManager = connection.getConnectionManager(); +// updateSeedNodes(connection, seedNodes(seedNode)); +// if (rejectedNode.equals(seedNode)) { +// assertFalse(connectionManager.nodeConnected(seedNode)); +// assertTrue(connectionManager.nodeConnected(discoverableNode)); +// } else { +// assertTrue(connectionManager.nodeConnected(seedNode)); +// assertFalse(connectionManager.nodeConnected(discoverableNode)); +// } +// assertTrue(connection.assertNoRunningConnections()); +// } +// } +// } +// } - public void testRemoteProfileIsUsedForRemoteCluster() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT, threadPool, - Settings.builder().put("cluster.name", "foobar").build()); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, - threadPool, Settings.builder().put("cluster.name", "foobar").build())) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, - profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, seedNodes(seedNode)); - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connection.assertNoRunningConnections()); - TransportFuture futureHandler = transportFuture(ClusterSearchShardsResponse::new); - TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.BULK) - .build(); - IllegalStateException ise = (IllegalStateException) expectThrows(SendRequestTransportException.class, () -> { - service.sendRequest(connectionManager.getConnection(discoverableNode), - ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), options, futureHandler); - futureHandler.txGet(); - }).getCause(); - assertEquals(ise.getMessage(), "can't select channel size is 0 for types: [RECOVERY, BULK, STATE]"); - - TransportFuture handler = transportFuture(ClusterSearchShardsResponse::new); - TransportRequestOptions ops = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.REG) - .build(); - service.sendRequest(connection.getConnection(), ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), - ops, handler); - handler.txGet(); - } - } - } - } - - public void testDiscoverSingleNode() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, - profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, seedNodes(seedNode)); - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connection.assertNoRunningConnections()); - } - } - } - } - - public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService incompatibleTransport = startTransport("incompat_seed_node", knownNodes, Version.fromString("2.0.0")); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - DiscoveryNode incompatibleSeedNode = incompatibleTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - knownNodes.add(incompatibleTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - List>> seedNodes = Arrays.asList( - Tuple.tuple(incompatibleSeedNode.toString(), () -> incompatibleSeedNode), - Tuple.tuple(seedNode.toString(), () -> seedNode)); - Collections.shuffle(seedNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes, service, Integer.MAX_VALUE, n -> true, null, profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, seedNodes); - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertFalse(connectionManager.nodeConnected(incompatibleSeedNode)); - assertTrue(connection.assertNoRunningConnections()); - } - } - } - } - - public void testNodeDisconnected() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT); - MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - DiscoveryNode spareNode = spareTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, seedNodes(seedNode)); - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertFalse(connectionManager.nodeConnected(spareNode)); - knownNodes.add(spareNode); - CountDownLatch latchDisconnect = new CountDownLatch(1); - CountDownLatch latchConnected = new CountDownLatch(1); - connectionManager.addListener(new TransportConnectionListener() { - @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { - if (node.equals(discoverableNode)) { - latchDisconnect.countDown(); - } - } - - @Override - public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) { - if (node.equals(spareNode)) { - latchConnected.countDown(); - } - } - }); - - discoverableTransport.close(); - // now make sure we try to connect again to other nodes once we got disconnected - assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS)); - assertTrue(latchConnected.await(10, TimeUnit.SECONDS)); - assertTrue(connectionManager.nodeConnected(spareNode)); - } - } - } - } - - public void testFilterDiscoveredNodes() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode; - Collections.shuffle(knownNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes(seedNode), service, Integer.MAX_VALUE, n -> n.equals(rejectedNode) == false, null, profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, seedNodes(seedNode)); - if (rejectedNode.equals(seedNode)) { - assertFalse(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - } else { - assertTrue(connectionManager.nodeConnected(seedNode)); - assertFalse(connectionManager.nodeConnected(discoverableNode)); - } - assertTrue(connection.assertNoRunningConnections()); - } - } - } - } private void updateSeedNodes( - final RemoteClusterConnection connection, final List>> seedNodes) throws Exception { + final RemoteClusterConnection connection, final List>> seedNodes) throws Exception { updateSeedNodes(connection, seedNodes, null); } private void updateSeedNodes( - final RemoteClusterConnection connection, - final List>> seedNodes, - final String proxyAddress) + final RemoteClusterConnection connection, + final List>> seedNodes, + final String proxyAddress) throws Exception { CountDownLatch latch = new CountDownLatch(1); AtomicReference exceptionAtomicReference = new AtomicReference<>(); @@ -425,28 +426,28 @@ private void updateSeedNodes( } } - public void testConnectWithIncompatibleTransports() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.fromString("2.0.0"))) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, - profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - expectThrows( - Exception.class, - () -> updateSeedNodes(connection, Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)))); - assertFalse(connectionManager.nodeConnected(seedNode)); - assertTrue(connection.assertNoRunningConnections()); - } - } - } - } +// public void testConnectWithIncompatibleTransports() throws Exception { +// List knownNodes = new CopyOnWriteArrayList<>(); +// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.fromString("2.0.0"))) { +// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); +// knownNodes.add(seedTransport.getLocalDiscoNode()); +// +// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { +// service.start(); +// service.acceptIncomingRequests(); +// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", +// Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, +// profile)) { +// ConnectionManager connectionManager = connection.getConnectionManager(); +// expectThrows( +// Exception.class, +// () -> updateSeedNodes(connection, Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)))); +// assertFalse(connectionManager.nodeConnected(seedNode)); +// assertTrue(connection.assertNoRunningConnections()); +// } +// } +// } +// } public void testRemoteConnectionVersionMatchesTransportConnectionVersion() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); @@ -491,7 +492,7 @@ public void sendRequest(long requestId, String action, TransportRequest request, service.start(); service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, connectionManager)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, connectionManager)) { PlainActionFuture.get(fut -> connection.ensureConnected(ActionListener.map(fut, x -> null))); assertThat(knownNodes, iterableWithSize(2)); assertThat(connection.getConnection(seedNode).getVersion(), equalTo(Version.CURRENT)); @@ -532,7 +533,7 @@ public void run() { CountDownLatch listenerCalled = new CountDownLatch(1); AtomicReference exceptionReference = new AtomicReference<>(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, profile)) { + seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, profile)) { ActionListener listener = ActionListener.wrap(x -> { listenerCalled.countDown(); fail("expected exception"); @@ -563,8 +564,8 @@ private static List>> seedNodes(final Disc return Collections.singletonList(Tuple.tuple(seedNodes[0].toString(), () -> seedNodes[0])); } else { return Arrays.stream(seedNodes) - .map(s -> Tuple.tuple(s.toString(), (Supplier)() -> s)) - .collect(Collectors.toList()); + .map(s -> Tuple.tuple(s.toString(), (Supplier) () -> s)) + .collect(Collectors.toList()); } } @@ -768,11 +769,11 @@ public void testGetConnectionInfo() throws Exception { public void testRemoteConnectionInfo() throws IOException { RemoteConnectionInfo stats = - new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(30), false); + new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(30), false); assertSerialization(stats); RemoteConnectionInfo stats1 = - new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 4, TimeValue.timeValueMinutes(30), true); + new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 4, TimeValue.timeValueMinutes(30), true); assertSerialization(stats1); assertNotEquals(stats, stats1); @@ -812,7 +813,7 @@ private static RemoteConnectionInfo assertSerialization(RemoteConnectionInfo inf public void testRenderConnectionInfoXContent() throws IOException { RemoteConnectionInfo stats = - new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(30), true); + new RemoteConnectionInfo("test_cluster", Arrays.asList("seed:1"), 4, 3, TimeValue.timeValueMinutes(30), true); stats = assertSerialization(stats); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); @@ -823,7 +824,7 @@ public void testRenderConnectionInfoXContent() throws IOException { "\"skip_unavailable\":true}}", Strings.toString(builder)); stats = new RemoteConnectionInfo( - "some_other_cluster", Arrays.asList("seed:1", "seed:2"), 2, 0, TimeValue.timeValueSeconds(30), false); + "some_other_cluster", Arrays.asList("seed:1", "seed:2"), 2, 0, TimeValue.timeValueSeconds(30), false); stats = assertSerialization(stats); builder = XContentFactory.jsonBuilder(); builder.startObject(); @@ -933,7 +934,7 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted try { final int numDiscoverableNodes = randomIntBetween(5, 20); List>> discoverableNodes = new ArrayList<>(numDiscoverableNodes); - for (int i = 0; i < numDiscoverableNodes; i++ ) { + for (int i = 0; i < numDiscoverableNodes; i++) { MockTransportService transportService = startTransport("discoverable_node" + i, knownNodes, Version.CURRENT); discoverableNodes.add(Tuple.tuple("discoverable_node" + i, transportService::getLocalDiscoNode)); discoverableTransports.add(transportService); @@ -1044,9 +1045,9 @@ public void testClusterNameIsChecked() throws Exception { assertTrue(connectionManager.nodeConnected(discoverableNode)); assertTrue(connection.assertNoRunningConnections()); List>> discoveryNodes = - Arrays.asList( - Tuple.tuple("other", otherClusterTransport::getLocalDiscoNode), - Tuple.tuple(seedNode.toString(), () -> seedNode)); + Arrays.asList( + Tuple.tuple("other", otherClusterTransport::getLocalDiscoNode), + Tuple.tuple(seedNode.toString(), () -> seedNode)); Collections.shuffle(discoveryNodes, random()); updateSeedNodes(connection, discoveryNodes); assertTrue(connectionManager.nodeConnected(seedNode)); @@ -1211,49 +1212,49 @@ public static Transport getProxyTransport(ThreadPool threadPool, Map { - Map proxyMapping = nodeMap.get(node.getAddress().toString()); - if (proxyMapping == null) { - throw new IllegalStateException("no proxy mapping for node: " + node); - } - DiscoveryNode proxyNode = proxyMapping.get(node.getName()); - if (proxyNode == null) { - // this is a seednode - lets pick one randomly - assertEquals("seed node must not have a port in the hostname: " + node.getHostName(), - -1, node.getHostName().lastIndexOf(':')); - assertTrue("missing hostname: " + node, proxyMapping.containsKey(node.getHostName())); - // route by seed hostname - proxyNode = proxyMapping.get(node.getHostName()); - } - t.openConnection(proxyNode, profile, ActionListener.delegateFailure(listener, - (delegatedListener, connection) -> delegatedListener.onResponse( - new Transport.Connection() { - @Override - public DiscoveryNode getNode() { - return node; - } + stubbableTransport.setDefaultConnectBehavior((t, node, profile, listener) -> { + Map proxyMapping = nodeMap.get(node.getAddress().toString()); + if (proxyMapping == null) { + throw new IllegalStateException("no proxy mapping for node: " + node); + } + DiscoveryNode proxyNode = proxyMapping.get(node.getName()); + if (proxyNode == null) { + // this is a seednode - lets pick one randomly + assertEquals("seed node must not have a port in the hostname: " + node.getHostName(), + -1, node.getHostName().lastIndexOf(':')); + assertTrue("missing hostname: " + node, proxyMapping.containsKey(node.getHostName())); + // route by seed hostname + proxyNode = proxyMapping.get(node.getHostName()); + } + t.openConnection(proxyNode, profile, ActionListener.delegateFailure(listener, + (delegatedListener, connection) -> delegatedListener.onResponse( + new Transport.Connection() { + @Override + public DiscoveryNode getNode() { + return node; + } - @Override - public void sendRequest(long requestId, String action, TransportRequest request, - TransportRequestOptions options) throws IOException { - connection.sendRequest(requestId, action, request, options); - } + @Override + public void sendRequest(long requestId, String action, TransportRequest request, + TransportRequestOptions options) throws IOException { + connection.sendRequest(requestId, action, request, options); + } - @Override - public void addCloseListener(ActionListener listener) { - connection.addCloseListener(listener); - } + @Override + public void addCloseListener(ActionListener listener) { + connection.addCloseListener(listener); + } - @Override - public boolean isClosed() { - return connection.isClosed(); - } + @Override + public boolean isClosed() { + return connection.isClosed(); + } - @Override - public void close() { - connection.close(); - } - }))); + @Override + public void close() { + connection.close(); + } + }))); }); return stubbableTransport; } diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index de780608217b8..53a361092fb38 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -20,16 +20,32 @@ package org.elasticsearch.transport; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; public class SniffConnectionStrategyTests extends ESTestCase { + private final String clusterAlias = "cluster-alias"; + private final ConnectionProfile profile = RemoteClusterService.buildConnectionProfileFromSettings(Settings.EMPTY, "cluster"); private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); @Override @@ -38,14 +54,220 @@ public void tearDown() throws Exception { ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); } - public void testThing() { - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); + private MockTransportService startTransport(String id, List knownNodes, Version version) { + return startTransport(id, knownNodes, version, threadPool); + } + + public static MockTransportService startTransport(String id, List knownNodes, Version version, ThreadPool threadPool) { + return startTransport(id, knownNodes, version, threadPool, Settings.EMPTY); + } + + public static MockTransportService startTransport(final String id, final List knownNodes, final Version version, + final ThreadPool threadPool, final Settings settings) { + boolean success = false; + final Settings s = Settings.builder().put(settings).put("node.name", id).build(); + ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(s); + MockTransportService newService = MockTransportService.createNewService(s, version, threadPool, null); + try { + newService.registerRequestHandler(ClusterStateAction.NAME, ThreadPool.Names.SAME, ClusterStateRequest::new, + (request, channel, task) -> { + DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); + for (DiscoveryNode node : knownNodes) { + builder.add(node); + } + ClusterState build = ClusterState.builder(clusterName).nodes(builder.build()).build(); + channel.sendResponse(new ClusterStateResponse(clusterName, build, false)); + }); + newService.start(); + newService.acceptIncomingRequests(); + success = true; + return newService; + } finally { + if (success == false) { + newService.close(); + } + } + } + + public void testSniffStrategyWillConnectToAndDiscoverNodes() { + List knownNodes = new CopyOnWriteArrayList<>(); + try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { + DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); + knownNodes.add(seedNode); + knownNodes.add(discoverableNode); + Collections.shuffle(knownNodes, random()); + + + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + localService.start(); + localService.acceptIncomingRequests(); + + List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); + ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); + try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, + null, 3, n -> true, seedNodes)) { + PlainActionFuture connectFuture = PlainActionFuture.newFuture(); + strategy.connect(connectFuture); + connectFuture.actionGet(); + + assertTrue(connectionManager.nodeConnected(seedNode)); + assertTrue(connectionManager.nodeConnected(discoverableNode)); + assertTrue(strategy.assertNoRunningConnections()); + } + + } + } + } + + public void testSniffStrategyWillConnectToMaxAllowedNodesAndOpenNewConnectionsOnDisconnect() throws Exception { + List knownNodes = new CopyOnWriteArrayList<>(); + try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService discoverableTransport1 = startTransport("discoverable_node", knownNodes, Version.CURRENT); + MockTransportService discoverableTransport2 = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { + DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); + DiscoveryNode discoverableNode1 = discoverableTransport1.getLocalDiscoNode(); + DiscoveryNode discoverableNode2 = discoverableTransport2.getLocalDiscoNode(); + knownNodes.add(seedNode); + knownNodes.add(discoverableNode1); + knownNodes.add(discoverableNode2); + Collections.shuffle(knownNodes, random()); + + + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + localService.start(); + localService.acceptIncomingRequests(); + + List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); + ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); + try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, + null, 2, n -> true, seedNodes)) { + PlainActionFuture connectFuture = PlainActionFuture.newFuture(); + strategy.connect(connectFuture); + connectFuture.actionGet(); - SniffConnectionStrategy strategy = new SniffConnectionStrategy("cluster", service, null, null, 6, null, null); + assertEquals(2, connectionManager.size()); + assertTrue(connectionManager.nodeConnected(seedNode)); + + // Assert that one of the discovered nodes is connected. After that, disconnect the connected + // discovered node and ensure the other discovered node is eventually connected + if (connectionManager.nodeConnected(discoverableNode1)) { + assertTrue(connectionManager.nodeConnected(discoverableNode1)); + discoverableTransport1.close(); + assertBusy(() -> assertTrue(connectionManager.nodeConnected(discoverableNode2))); + } else { + assertTrue(connectionManager.nodeConnected(discoverableNode2)); + discoverableTransport2.close(); + assertBusy(() -> assertTrue(connectionManager.nodeConnected(discoverableNode1))); + } + } + } } + } + + public void testDiscoverWithSingleIncompatibleSeedNode() { + List knownNodes = new CopyOnWriteArrayList<>(); + Version incompatibleVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion(); + try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService incompatibleSeedTransport = startTransport("discoverable_node", knownNodes, incompatibleVersion); + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { + DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); + DiscoveryNode incompatibleSeedNode = incompatibleSeedTransport.getLocalDiscoNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); + knownNodes.add(seedNode); + knownNodes.add(incompatibleSeedNode); + knownNodes.add(discoverableNode); + Collections.shuffle(knownNodes, random()); + + + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + localService.start(); + localService.acceptIncomingRequests(); + + List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); + ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); + try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, + null, 3, n -> true, seedNodes)) { + PlainActionFuture connectFuture = PlainActionFuture.newFuture(); + strategy.connect(connectFuture); + connectFuture.actionGet(); + + assertEquals(2, connectionManager.size()); + assertTrue(connectionManager.nodeConnected(seedNode)); + assertTrue(connectionManager.nodeConnected(discoverableNode)); + assertFalse(connectionManager.nodeConnected(incompatibleSeedNode)); + assertTrue(strategy.assertNoRunningConnections()); + } + } + } + } + public void testConnectFailsWithIncompatibleNodes() { + List knownNodes = new CopyOnWriteArrayList<>(); + Version incompatibleVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion(); + try (MockTransportService incompatibleSeedTransport = startTransport("seed_node", knownNodes, incompatibleVersion)) { + DiscoveryNode incompatibleSeedNode = incompatibleSeedTransport.getLocalDiscoNode(); + knownNodes.add(incompatibleSeedNode); + + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + localService.start(); + localService.acceptIncomingRequests(); + + Tuple> tuple = Tuple.tuple(incompatibleSeedNode.toString(), () -> incompatibleSeedNode); + List>> seedNodes = Collections.singletonList(tuple); + ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); + try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, + null, 3, n -> true, seedNodes)) { + PlainActionFuture connectFuture = PlainActionFuture.newFuture(); + strategy.connect(connectFuture); + + expectThrows(Exception.class, connectFuture::actionGet); + assertFalse(connectionManager.nodeConnected(incompatibleSeedNode)); + assertTrue(strategy.assertNoRunningConnections()); + } + } + } } + public void testFilterNodesWithNodePredicate() { + List knownNodes = new CopyOnWriteArrayList<>(); + try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { + DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); + knownNodes.add(seedNode); + knownNodes.add(discoverableNode); + DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode; + Collections.shuffle(knownNodes, random()); + + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + localService.start(); + localService.acceptIncomingRequests(); + + List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); + ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); + try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, + null, 3, n -> n.equals(rejectedNode) == false, seedNodes)) { + PlainActionFuture connectFuture = PlainActionFuture.newFuture(); + strategy.connect(connectFuture); + connectFuture.actionGet(); + + if (rejectedNode.equals(seedNode)) { + assertFalse(connectionManager.nodeConnected(seedNode)); + assertTrue(connectionManager.nodeConnected(discoverableNode)); + } else { + assertTrue(connectionManager.nodeConnected(seedNode)); + assertFalse(connectionManager.nodeConnected(discoverableNode)); + } + assertTrue(strategy.assertNoRunningConnections()); + } + } + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index cb509dfbf9d73..02688a635f048 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -99,6 +99,10 @@ public List> getSettings() { } } + public static MockTransportService createNewService(Settings settings, Version version, ThreadPool threadPool) { + return createNewService(settings, version, threadPool, null); + } + public static MockTransportService createNewService(Settings settings, Version version, ThreadPool threadPool, @Nullable ClusterSettings clusterSettings) { MockNioTransport mockTransport = newMockTransport(settings, version, threadPool); From bcae80fa6ed01d6897502bf0a404020c5db5c4b7 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 27 Sep 2019 13:52:34 -0600 Subject: [PATCH 07/12] More testing --- .../RemoteClusterConnectionTests.java | 414 ------------------ .../SniffConnectionStrategyTests.java | 142 +++++- 2 files changed, 130 insertions(+), 426 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 5bb2aa22a9dd1..5c79f8a7fe682 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -180,226 +180,6 @@ public static MockTransportService startTransport( } } -// public void testRemoteProfileIsUsedForLocalCluster() throws Exception { -// List knownNodes = new CopyOnWriteArrayList<>(); -// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); -// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { -// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); -// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); -// knownNodes.add(seedTransport.getLocalDiscoNode()); -// knownNodes.add(discoverableTransport.getLocalDiscoNode()); -// Collections.shuffle(knownNodes, random()); -// -// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { -// service.start(); -// service.acceptIncomingRequests(); -// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", -// Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, -// profile)) { -// ConnectionManager connectionManager = connection.getConnectionManager(); -// updateSeedNodes(connection, seedNodes(seedNode)); -// assertTrue(connectionManager.nodeConnected(seedNode)); -// assertTrue(connectionManager.nodeConnected(discoverableNode)); -// assertTrue(connection.assertNoRunningConnections()); -// TransportFuture futureHandler = transportFuture(ClusterSearchShardsResponse::new); -// TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.BULK) -// .build(); -// IllegalStateException ise = (IllegalStateException) expectThrows(SendRequestTransportException.class, () -> { -// service.sendRequest(connectionManager.getConnection(discoverableNode), -// ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), options, futureHandler); -// futureHandler.txGet(); -// }).getCause(); -// assertEquals(ise.getMessage(), "can't select channel size is 0 for types: [RECOVERY, BULK, STATE]"); -// } -// } -// } -// } -// -// public void testRemoteProfileIsUsedForRemoteCluster() throws Exception { -// List knownNodes = new CopyOnWriteArrayList<>(); -// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT, threadPool, -// Settings.builder().put("cluster.name", "foobar").build()); -// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, -// threadPool, Settings.builder().put("cluster.name", "foobar").build())) { -// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); -// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); -// knownNodes.add(seedTransport.getLocalDiscoNode()); -// knownNodes.add(discoverableTransport.getLocalDiscoNode()); -// Collections.shuffle(knownNodes, random()); -// -// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { -// service.start(); -// service.acceptIncomingRequests(); -// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", -// Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, -// profile)) { -// ConnectionManager connectionManager = connection.getConnectionManager(); -// updateSeedNodes(connection, seedNodes(seedNode)); -// assertTrue(connectionManager.nodeConnected(seedNode)); -// assertTrue(connectionManager.nodeConnected(discoverableNode)); -// assertTrue(connection.assertNoRunningConnections()); -// TransportFuture futureHandler = transportFuture(ClusterSearchShardsResponse::new); -// TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.BULK) -// .build(); -// IllegalStateException ise = (IllegalStateException) expectThrows(SendRequestTransportException.class, () -> { -// service.sendRequest(connectionManager.getConnection(discoverableNode), -// ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), options, futureHandler); -// futureHandler.txGet(); -// }).getCause(); -// assertEquals(ise.getMessage(), "can't select channel size is 0 for types: [RECOVERY, BULK, STATE]"); -// -// TransportFuture handler = transportFuture(ClusterSearchShardsResponse::new); -// TransportRequestOptions ops = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.REG) -// .build(); -// service.sendRequest(connection.getConnection(), ClusterSearchShardsAction.NAME, new ClusterSearchShardsRequest(), -// ops, handler); -// handler.txGet(); -// } -// } -// } -// } -// -// public void testDiscoverSingleNode() throws Exception { -// List knownNodes = new CopyOnWriteArrayList<>(); -// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); -// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { -// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); -// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); -// knownNodes.add(seedTransport.getLocalDiscoNode()); -// knownNodes.add(discoverableTransport.getLocalDiscoNode()); -// Collections.shuffle(knownNodes, random()); -// -// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { -// service.start(); -// service.acceptIncomingRequests(); -// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", -// Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, -// profile)) { -// ConnectionManager connectionManager = connection.getConnectionManager(); -// updateSeedNodes(connection, seedNodes(seedNode)); -// assertTrue(connectionManager.nodeConnected(seedNode)); -// assertTrue(connectionManager.nodeConnected(discoverableNode)); -// assertTrue(connection.assertNoRunningConnections()); -// } -// } -// } -// } - -// public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception { -// List knownNodes = new CopyOnWriteArrayList<>(); -// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); -// MockTransportService incompatibleTransport = startTransport("incompat_seed_node", knownNodes, Version.fromString("2.0.0")); -// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { -// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); -// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); -// DiscoveryNode incompatibleSeedNode = incompatibleTransport.getLocalDiscoNode(); -// knownNodes.add(seedTransport.getLocalDiscoNode()); -// knownNodes.add(discoverableTransport.getLocalDiscoNode()); -// knownNodes.add(incompatibleTransport.getLocalDiscoNode()); -// Collections.shuffle(knownNodes, random()); -// List>> seedNodes = Arrays.asList( -// Tuple.tuple(incompatibleSeedNode.toString(), () -> incompatibleSeedNode), -// Tuple.tuple(seedNode.toString(), () -> seedNode)); -// Collections.shuffle(seedNodes, random()); -// -// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { -// service.start(); -// service.acceptIncomingRequests(); -// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", -// seedNodes, service, Integer.MAX_VALUE, n -> true, null, profile)) { -// ConnectionManager connectionManager = connection.getConnectionManager(); -// updateSeedNodes(connection, seedNodes); -// assertTrue(connectionManager.nodeConnected(seedNode)); -// assertTrue(connectionManager.nodeConnected(discoverableNode)); -// assertFalse(connectionManager.nodeConnected(incompatibleSeedNode)); -// assertTrue(connection.assertNoRunningConnections()); -// } -// } -// } -// } - -// public void testNodeDisconnected() throws Exception { -// List knownNodes = new CopyOnWriteArrayList<>(); -// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); -// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT); -// MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) { -// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); -// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); -// DiscoveryNode spareNode = spareTransport.getLocalDiscoNode(); -// knownNodes.add(seedTransport.getLocalDiscoNode()); -// knownNodes.add(discoverableTransport.getLocalDiscoNode()); -// Collections.shuffle(knownNodes, random()); -// -// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { -// service.start(); -// service.acceptIncomingRequests(); -// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", -// seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, profile)) { -// ConnectionManager connectionManager = connection.getConnectionManager(); -// updateSeedNodes(connection, seedNodes(seedNode)); -// assertTrue(connectionManager.nodeConnected(seedNode)); -// assertTrue(connectionManager.nodeConnected(discoverableNode)); -// assertFalse(connectionManager.nodeConnected(spareNode)); -// knownNodes.add(spareNode); -// CountDownLatch latchDisconnect = new CountDownLatch(1); -// CountDownLatch latchConnected = new CountDownLatch(1); -// connectionManager.addListener(new TransportConnectionListener() { -// @Override -// public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { -// if (node.equals(discoverableNode)) { -// latchDisconnect.countDown(); -// } -// } -// -// @Override -// public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) { -// if (node.equals(spareNode)) { -// latchConnected.countDown(); -// } -// } -// }); -// -// discoverableTransport.close(); -// // now make sure we try to connect again to other nodes once we got disconnected -// assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS)); -// assertTrue(latchConnected.await(10, TimeUnit.SECONDS)); -// assertTrue(connectionManager.nodeConnected(spareNode)); -// } -// } -// } -// } - -// public void testFilterDiscoveredNodes() throws Exception { -// List knownNodes = new CopyOnWriteArrayList<>(); -// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); -// MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { -// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); -// DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); -// knownNodes.add(seedTransport.getLocalDiscoNode()); -// knownNodes.add(discoverableTransport.getLocalDiscoNode()); -// DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode; -// Collections.shuffle(knownNodes, random()); -// -// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { -// service.start(); -// service.acceptIncomingRequests(); -// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", -// seedNodes(seedNode), service, Integer.MAX_VALUE, n -> n.equals(rejectedNode) == false, null, profile)) { -// ConnectionManager connectionManager = connection.getConnectionManager(); -// updateSeedNodes(connection, seedNodes(seedNode)); -// if (rejectedNode.equals(seedNode)) { -// assertFalse(connectionManager.nodeConnected(seedNode)); -// assertTrue(connectionManager.nodeConnected(discoverableNode)); -// } else { -// assertTrue(connectionManager.nodeConnected(seedNode)); -// assertFalse(connectionManager.nodeConnected(discoverableNode)); -// } -// assertTrue(connection.assertNoRunningConnections()); -// } -// } -// } -// } - private void updateSeedNodes( final RemoteClusterConnection connection, final List>> seedNodes) throws Exception { updateSeedNodes(connection, seedNodes, null); @@ -426,82 +206,6 @@ private void updateSeedNodes( } } -// public void testConnectWithIncompatibleTransports() throws Exception { -// List knownNodes = new CopyOnWriteArrayList<>(); -// try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.fromString("2.0.0"))) { -// DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); -// knownNodes.add(seedTransport.getLocalDiscoNode()); -// -// try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { -// service.start(); -// service.acceptIncomingRequests(); -// try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", -// Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)), service, Integer.MAX_VALUE, n -> true, null, -// profile)) { -// ConnectionManager connectionManager = connection.getConnectionManager(); -// expectThrows( -// Exception.class, -// () -> updateSeedNodes(connection, Arrays.asList(Tuple.tuple(seedNode.toString(), () -> seedNode)))); -// assertFalse(connectionManager.nodeConnected(seedNode)); -// assertTrue(connection.assertNoRunningConnections()); -// } -// } -// } -// } - - public void testRemoteConnectionVersionMatchesTransportConnectionVersion() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - final Version previousVersion = randomValueOtherThan(Version.CURRENT, () -> VersionUtils.randomVersionBetween(random(), - Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT)); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, previousVersion)) { - - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - assertThat(seedNode, notNullValue()); - knownNodes.add(seedNode); - - DiscoveryNode oldVersionNode = discoverableTransport.getLocalDiscoNode(); - assertThat(oldVersionNode, notNullValue()); - knownNodes.add(oldVersionNode); - - assertThat(seedNode.getVersion(), not(equalTo(oldVersionNode.getVersion()))); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - final Transport.Connection seedConnection = new CloseableConnection() { - @Override - public DiscoveryNode getNode() { - return seedNode; - } - - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) - throws IOException, TransportException { - // no-op - } - }; - - ConnectionManager delegate = new ConnectionManager(Settings.EMPTY, service.transport); - StubbableConnectionManager connectionManager = new StubbableConnectionManager(delegate, Settings.EMPTY, service.transport); - - connectionManager.addGetConnectionBehavior(seedNode.getAddress(), (cm, discoveryNode) -> { - if (discoveryNode == seedNode) { - return seedConnection; - } - return cm.getConnection(discoveryNode); - }); - - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, connectionManager)) { - PlainActionFuture.get(fut -> connection.ensureConnected(ActionListener.map(fut, x -> null))); - assertThat(knownNodes, iterableWithSize(2)); - assertThat(connection.getConnection(seedNode).getVersion(), equalTo(Version.CURRENT)); - assertThat(connection.getConnection(oldVersionNode).getVersion(), equalTo(previousVersion)); - } - } - } - } - @SuppressForbidden(reason = "calls getLocalHost here but it's fine in this case") public void testSlowNodeCanBeCancelled() throws IOException, InterruptedException { try (ServerSocket socket = new MockServerSocket()) { @@ -835,61 +539,6 @@ public void testRenderConnectionInfoXContent() throws IOException { "\"skip_unavailable\":false}}", Strings.toString(builder)); } - public void testEnsureConnected() throws IOException, InterruptedException { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - assertFalse(connectionManager.nodeConnected(seedNode)); - assertFalse(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connection.assertNoRunningConnections()); - CountDownLatch latch = new CountDownLatch(1); - connection.ensureConnected(new LatchedActionListener<>(new ActionListener() { - @Override - public void onResponse(Void aVoid) { - } - - @Override - public void onFailure(Exception e) { - throw new AssertionError(e); - } - }, latch)); - latch.await(); - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connection.assertNoRunningConnections()); - - // exec again we are already connected - connection.ensureConnected(new LatchedActionListener<>(new ActionListener() { - @Override - public void onResponse(Void aVoid) { - } - - @Override - public void onFailure(Exception e) { - throw new AssertionError(e); - } - }, latch)); - latch.await(); - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connection.assertNoRunningConnections()); - } - } - } - } - public void testCollectNodes() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT)) { @@ -1164,47 +813,6 @@ public void testLazyResolveTransportAddress() throws Exception { } } - public void testProxyMode() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("node_0", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("node_1", knownNodes, Version.CURRENT)) { - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - final String proxyAddress = "1.1.1.1:99"; - Map nodes = new HashMap<>(); - nodes.put("node_0", seedTransport.getLocalDiscoNode()); - nodes.put("node_1", discoverableTransport.getLocalDiscoNode()); - Transport mockTcpTransport = getProxyTransport(threadPool, Collections.singletonMap(proxyAddress, nodes)); - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, mockTcpTransport, Version.CURRENT, - threadPool, null, Collections.emptySet())) { - service.start(); - service.acceptIncomingRequests(); - Tuple> seedSupplier = Tuple.tuple("node_0", () -> - RemoteClusterAware.buildSeedNode("some-remote-cluster", "node_0:" + randomIntBetween(1, 10000), true)); - assertEquals("node_0", seedSupplier.v2().get().getAttributes().get("server_name")); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - Arrays.asList(seedSupplier), service, Integer.MAX_VALUE, n -> true, proxyAddress, profile)) { - updateSeedNodes(connection, Arrays.asList(seedSupplier), proxyAddress); - assertEquals(2, connection.getNumNodesConnected()); - assertNotNull(connection.getConnection(discoverableTransport.getLocalDiscoNode())); - assertNotNull(connection.getConnection(seedTransport.getLocalDiscoNode())); - assertEquals(proxyAddress, connection.getConnection(seedTransport.getLocalDiscoNode()) - .getNode().getAddress().toString()); - assertEquals(proxyAddress, connection.getConnection(discoverableTransport.getLocalDiscoNode()) - .getNode().getAddress().toString()); - connection.getConnectionManager().disconnectFromNode(knownNodes.get(0)); - // ensure we reconnect - assertBusy(() -> { - assertEquals(2, connection.getNumNodesConnected()); - }); - discoverableTransport.close(); - seedTransport.close(); - } - } - } - } - public static Transport getProxyTransport(ThreadPool threadPool, Map> nodeMap) { if (nodeMap.isEmpty()) { throw new IllegalArgumentException("nodeMap must be non-empty"); @@ -1258,26 +866,4 @@ public void close() { }); return stubbableTransport; } - - private static TransportFuture transportFuture(Writeable.Reader reader) { - return new TransportFuture<>(new TransportResponseHandler<>() { - @Override - public void handleResponse(V response) { - } - - @Override - public void handleException(final TransportException exp) { - } - - @Override - public String executor() { - return ThreadPool.Names.SAME; - } - - @Override - public V read(StreamInput in) throws IOException { - return reader.read(in); - } - }); - } } diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index 53a361092fb38..e58d0fb5627f1 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -20,6 +20,8 @@ package org.elasticsearch.transport; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -28,18 +30,26 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.test.transport.StubbableConnectionManager; +import org.elasticsearch.test.transport.StubbableTransport; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import java.io.IOException; +import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; public class SniffConnectionStrategyTests extends ESTestCase { @@ -67,7 +77,7 @@ public static MockTransportService startTransport(final String id, final List { @@ -93,8 +103,8 @@ public void testSniffStrategyWillConnectToAndDiscoverNodes() { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); + DiscoveryNode seedNode = seedTransport.getLocalNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalNode(); knownNodes.add(seedNode); knownNodes.add(discoverableNode); Collections.shuffle(knownNodes, random()); @@ -127,9 +137,9 @@ public void testSniffStrategyWillConnectToMaxAllowedNodesAndOpenNewConnectionsOn try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); MockTransportService discoverableTransport1 = startTransport("discoverable_node", knownNodes, Version.CURRENT); MockTransportService discoverableTransport2 = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode1 = discoverableTransport1.getLocalDiscoNode(); - DiscoveryNode discoverableNode2 = discoverableTransport2.getLocalDiscoNode(); + DiscoveryNode seedNode = seedTransport.getLocalNode(); + DiscoveryNode discoverableNode1 = discoverableTransport1.getLocalNode(); + DiscoveryNode discoverableNode2 = discoverableTransport2.getLocalNode(); knownNodes.add(seedNode); knownNodes.add(discoverableNode1); knownNodes.add(discoverableNode2); @@ -174,9 +184,9 @@ public void testDiscoverWithSingleIncompatibleSeedNode() { try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); MockTransportService incompatibleSeedTransport = startTransport("discoverable_node", knownNodes, incompatibleVersion); MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode incompatibleSeedNode = incompatibleSeedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); + DiscoveryNode seedNode = seedTransport.getLocalNode(); + DiscoveryNode incompatibleSeedNode = incompatibleSeedTransport.getLocalNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalNode(); knownNodes.add(seedNode); knownNodes.add(incompatibleSeedNode); knownNodes.add(discoverableNode); @@ -210,7 +220,7 @@ public void testConnectFailsWithIncompatibleNodes() { List knownNodes = new CopyOnWriteArrayList<>(); Version incompatibleVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion(); try (MockTransportService incompatibleSeedTransport = startTransport("seed_node", knownNodes, incompatibleVersion)) { - DiscoveryNode incompatibleSeedNode = incompatibleSeedTransport.getLocalDiscoNode(); + DiscoveryNode incompatibleSeedNode = incompatibleSeedTransport.getLocalNode(); knownNodes.add(incompatibleSeedNode); try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { @@ -238,8 +248,8 @@ public void testFilterNodesWithNodePredicate() { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); + DiscoveryNode seedNode = seedTransport.getLocalNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalNode(); knownNodes.add(seedNode); knownNodes.add(discoverableNode); DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode; @@ -270,4 +280,112 @@ public void testFilterNodesWithNodePredicate() { } } } + + public void testMultipleCallsToConnectEnsuresConnection() { + List knownNodes = new CopyOnWriteArrayList<>(); + try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { + DiscoveryNode seedNode = seedTransport.getLocalNode(); + DiscoveryNode discoverableNode = discoverableTransport.getLocalNode(); + knownNodes.add(seedNode); + knownNodes.add(discoverableNode); + Collections.shuffle(knownNodes, random()); + + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + localService.start(); + localService.acceptIncomingRequests(); + + List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); + ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); + try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, + null, 3, n -> true, seedNodes)) { + assertFalse(connectionManager.nodeConnected(seedNode)); + assertFalse(connectionManager.nodeConnected(discoverableNode)); + assertTrue(strategy.assertNoRunningConnections()); + + PlainActionFuture connectFuture = PlainActionFuture.newFuture(); + strategy.connect(connectFuture); + connectFuture.actionGet(); + + assertTrue(connectionManager.nodeConnected(seedNode)); + assertTrue(connectionManager.nodeConnected(discoverableNode)); + assertTrue(strategy.assertNoRunningConnections()); + + // exec again we are already connected + PlainActionFuture ensureConnectFuture = PlainActionFuture.newFuture(); + strategy.connect(ensureConnectFuture); + ensureConnectFuture.actionGet(); + + assertTrue(connectionManager.nodeConnected(seedNode)); + assertTrue(connectionManager.nodeConnected(discoverableNode)); + assertTrue(strategy.assertNoRunningConnections()); + } + } + } + } + + public void testConfiguredProxyAddressModeWillReplaceNodeAddress() { + List knownNodes = new CopyOnWriteArrayList<>(); + try (MockTransportService accessible = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService unresponsive1 = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool); + MockTransportService unresponsive2 = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + // We start in order to get a valid address + port, but do not start accepting connections as we + // will not actually connect to these transports + unresponsive1.start(); + unresponsive2.start(); + DiscoveryNode accessibleNode = accessible.getLocalNode(); + DiscoveryNode discoverableNode = unresponsive2.getLocalNode(); + + // Use the address for the node that will not respond + DiscoveryNode unaddressableSeedNode = new DiscoveryNode(accessibleNode.getName(), accessibleNode.getId(), + accessibleNode.getEphemeralId(), accessibleNode.getHostName(), accessibleNode.getHostAddress(), + unresponsive1.getLocalNode().getAddress(), accessibleNode.getAttributes(), accessibleNode.getRoles(), + accessibleNode.getVersion()); + + knownNodes.add(unaddressableSeedNode); + knownNodes.add(discoverableNode); + Collections.shuffle(knownNodes, random()); + + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + localService.start(); + localService.acceptIncomingRequests(); + + StubbableTransport transport = new StubbableTransport(localService.transport); + AtomicReference discoverableNodeAddress = new AtomicReference<>(); + transport.setDefaultConnectBehavior((delegate, node, profile, listener) -> { + if (node.equals(discoverableNode)) { + // Do not actually try to connect because the node will not respond. Just capture the + // address for later assertion + discoverableNodeAddress.set(node.getAddress()); + listener.onFailure(new ConnectTransportException(node, "general failure")); + } else { + delegate.openConnection(node, profile, listener); + } + }); + + Tuple> tuple = Tuple.tuple(accessibleNode.toString(), () -> unaddressableSeedNode); + List>> seedNodes = Collections.singletonList(tuple); + TransportAddress proxyAddress = accessibleNode.getAddress(); + ConnectionManager connectionManager = new ConnectionManager(profile, transport); + try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, + proxyAddress.toString(), 3, n -> true, seedNodes)) { + assertFalse(connectionManager.nodeConnected(unaddressableSeedNode)); + assertFalse(connectionManager.nodeConnected(discoverableNode)); + assertTrue(strategy.assertNoRunningConnections()); + + PlainActionFuture connectFuture = PlainActionFuture.newFuture(); + strategy.connect(connectFuture); + connectFuture.actionGet(); + + assertTrue(connectionManager.nodeConnected(unaddressableSeedNode)); + // Connection to discoverable will fail due to the stubbable transport + assertFalse(connectionManager.nodeConnected(discoverableNode)); + assertEquals(proxyAddress, discoverableNodeAddress.get()); + assertTrue(strategy.assertNoRunningConnections()); + } + } + } + } } From 690dca6c94afeb96078fb02dceb359d3828c5630 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 27 Sep 2019 14:37:25 -0600 Subject: [PATCH 08/12] Work on tests --- .../RemoteClusterConnectionTests.java | 6 -- .../SniffConnectionStrategyTests.java | 93 +++++++++++++------ 2 files changed, 65 insertions(+), 34 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 5c79f8a7fe682..277f36ff91a7d 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -22,7 +22,6 @@ import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; @@ -44,7 +43,6 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; @@ -58,7 +56,6 @@ import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.test.transport.StubbableConnectionManager; import org.elasticsearch.test.transport.StubbableTransport; @@ -73,7 +70,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.BrokenBarrierException; @@ -96,8 +92,6 @@ import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.iterableWithSize; -import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.Matchers.startsWith; diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index e58d0fb5627f1..2ccd1a0637921 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -20,8 +20,6 @@ package org.elasticsearch.transport; import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -30,27 +28,23 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.test.transport.StubbableConnectionManager; import org.elasticsearch.test.transport.StubbableTransport; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import java.io.IOException; -import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import java.util.stream.Collectors; public class SniffConnectionStrategyTests extends ESTestCase { @@ -65,17 +59,17 @@ public void tearDown() throws Exception { } private MockTransportService startTransport(String id, List knownNodes, Version version) { - return startTransport(id, knownNodes, version, threadPool); + return startTransport(id, knownNodes, version, Settings.EMPTY); } - public static MockTransportService startTransport(String id, List knownNodes, Version version, ThreadPool threadPool) { - return startTransport(id, knownNodes, version, threadPool, Settings.EMPTY); - } - - public static MockTransportService startTransport(final String id, final List knownNodes, final Version version, - final ThreadPool threadPool, final Settings settings) { + public MockTransportService startTransport(final String id, final List knownNodes, final Version version, + final Settings settings) { boolean success = false; - final Settings s = Settings.builder().put(settings).put("node.name", id).build(); + final Settings s = Settings.builder() + .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterAlias) + .put("node.name", id) + .put(settings) + .build(); ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(s); MockTransportService newService = MockTransportService.createNewService(s, version, threadPool); try { @@ -114,11 +108,10 @@ public void testSniffStrategyWillConnectToAndDiscoverNodes() { localService.start(); localService.acceptIncomingRequests(); - List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, - null, 3, n -> true, seedNodes)) { + null, 3, n -> true, seedNodes(seedNode))) { PlainActionFuture connectFuture = PlainActionFuture.newFuture(); strategy.connect(connectFuture); connectFuture.actionGet(); @@ -150,11 +143,10 @@ public void testSniffStrategyWillConnectToMaxAllowedNodesAndOpenNewConnectionsOn localService.start(); localService.acceptIncomingRequests(); - List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, - null, 2, n -> true, seedNodes)) { + null, 2, n -> true, seedNodes(seedNode))) { PlainActionFuture connectFuture = PlainActionFuture.newFuture(); strategy.connect(connectFuture); connectFuture.actionGet(); @@ -197,11 +189,10 @@ public void testDiscoverWithSingleIncompatibleSeedNode() { localService.start(); localService.acceptIncomingRequests(); - List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, - null, 3, n -> true, seedNodes)) { + null, 3, n -> true, seedNodes(seedNode))) { PlainActionFuture connectFuture = PlainActionFuture.newFuture(); strategy.connect(connectFuture); connectFuture.actionGet(); @@ -227,12 +218,10 @@ public void testConnectFailsWithIncompatibleNodes() { localService.start(); localService.acceptIncomingRequests(); - Tuple> tuple = Tuple.tuple(incompatibleSeedNode.toString(), () -> incompatibleSeedNode); - List>> seedNodes = Collections.singletonList(tuple); ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, - null, 3, n -> true, seedNodes)) { + null, 3, n -> true, seedNodes(incompatibleSeedNode))) { PlainActionFuture connectFuture = PlainActionFuture.newFuture(); strategy.connect(connectFuture); @@ -259,11 +248,10 @@ public void testFilterNodesWithNodePredicate() { localService.start(); localService.acceptIncomingRequests(); - List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, - null, 3, n -> n.equals(rejectedNode) == false, seedNodes)) { + null, 3, n -> n.equals(rejectedNode) == false, seedNodes(seedNode))) { PlainActionFuture connectFuture = PlainActionFuture.newFuture(); strategy.connect(connectFuture); connectFuture.actionGet(); @@ -281,6 +269,44 @@ public void testFilterNodesWithNodePredicate() { } } + public void testClusterNameIsValidated() { + List knownNodes = new CopyOnWriteArrayList<>(); + List otherClusterKnownNodes = new CopyOnWriteArrayList<>(); + + Settings otherSettings = Settings.builder().put("cluster.name", "otherCluster").build(); + + try (MockTransportService seed = startTransport("seed", knownNodes, Version.CURRENT); + MockTransportService discoverable = startTransport("discoverable", knownNodes, Version.CURRENT); + MockTransportService otherSeed = startTransport("other_seed", knownNodes, Version.CURRENT, otherSettings); + MockTransportService otherDiscoverable = startTransport("other_discoverable", knownNodes, Version.CURRENT, otherSettings)) { + DiscoveryNode seedNode = seed.getLocalNode(); + DiscoveryNode discoverableNode = discoverable.getLocalNode(); + DiscoveryNode otherDiscoverableNode = otherDiscoverable.getLocalNode(); + knownNodes.add(seedNode); + knownNodes.add(discoverableNode); + knownNodes.add(otherDiscoverableNode); + otherClusterKnownNodes.add(otherSeed.getLocalDiscoNode()); + otherClusterKnownNodes.add(otherDiscoverable.getLocalDiscoNode()); + Collections.shuffle(knownNodes, random()); + Collections.shuffle(otherClusterKnownNodes, random()); + + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { + localService.start(); + localService.acceptIncomingRequests(); + + ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); + try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); + SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, + null, 3, n -> true, seedNodes(seedNode))) { + assertTrue(connectionManager.nodeConnected(seedNode)); + assertTrue(connectionManager.nodeConnected(discoverableNode)); + assertFalse(connectionManager.nodeConnected(otherDiscoverableNode)); + assertTrue(strategy.assertNoRunningConnections()); + } + } + } + } + public void testMultipleCallsToConnectEnsuresConnection() { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); @@ -295,11 +321,10 @@ public void testMultipleCallsToConnectEnsuresConnection() { localService.start(); localService.acceptIncomingRequests(); - List>> seedNodes = Collections.singletonList(Tuple.tuple(seedNode.toString(), () -> seedNode)); ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, - null, 3, n -> true, seedNodes)) { + null, 3, n -> true, seedNodes(seedNode))) { assertFalse(connectionManager.nodeConnected(seedNode)); assertFalse(connectionManager.nodeConnected(discoverableNode)); assertTrue(strategy.assertNoRunningConnections()); @@ -388,4 +413,16 @@ public void testConfiguredProxyAddressModeWillReplaceNodeAddress() { } } } + + private static List>> seedNodes(final DiscoveryNode... seedNodes) { + if (seedNodes.length == 0) { + return Collections.emptyList(); + } else if (seedNodes.length == 1) { + return Collections.singletonList(Tuple.tuple(seedNodes[0].toString(), () -> seedNodes[0])); + } else { + return Arrays.stream(seedNodes) + .map(s -> Tuple.tuple(s.toString(), (Supplier) () -> s)) + .collect(Collectors.toList()); + } + } } From 7b79c8e27b46400eb258f565f428d89981e91990 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 27 Sep 2019 14:46:40 -0600 Subject: [PATCH 09/12] WIP --- .../SniffConnectionStrategyTests.java | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index 2ccd1a0637921..75fa9680a97d3 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -46,6 +46,11 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.startsWith; + public class SniffConnectionStrategyTests extends ESTestCase { private final String clusterAlias = "cluster-alias"; @@ -271,24 +276,16 @@ public void testFilterNodesWithNodePredicate() { public void testClusterNameIsValidated() { List knownNodes = new CopyOnWriteArrayList<>(); - List otherClusterKnownNodes = new CopyOnWriteArrayList<>(); Settings otherSettings = Settings.builder().put("cluster.name", "otherCluster").build(); - try (MockTransportService seed = startTransport("seed", knownNodes, Version.CURRENT); - MockTransportService discoverable = startTransport("discoverable", knownNodes, Version.CURRENT); - MockTransportService otherSeed = startTransport("other_seed", knownNodes, Version.CURRENT, otherSettings); + try (MockTransportService otherSeed = startTransport("other_seed", knownNodes, Version.CURRENT, otherSettings); MockTransportService otherDiscoverable = startTransport("other_discoverable", knownNodes, Version.CURRENT, otherSettings)) { - DiscoveryNode seedNode = seed.getLocalNode(); - DiscoveryNode discoverableNode = discoverable.getLocalNode(); + DiscoveryNode otherSeedNode = otherSeed.getLocalNode(); DiscoveryNode otherDiscoverableNode = otherDiscoverable.getLocalNode(); - knownNodes.add(seedNode); - knownNodes.add(discoverableNode); + knownNodes.add(otherSeedNode); knownNodes.add(otherDiscoverableNode); - otherClusterKnownNodes.add(otherSeed.getLocalDiscoNode()); - otherClusterKnownNodes.add(otherDiscoverable.getLocalDiscoNode()); Collections.shuffle(knownNodes, random()); - Collections.shuffle(otherClusterKnownNodes, random()); try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { localService.start(); @@ -297,9 +294,17 @@ public void testClusterNameIsValidated() { ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, - null, 3, n -> true, seedNodes(seedNode))) { - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); + null, 3, n -> true, seedNodes(otherSeedNode))) { + PlainActionFuture connectFuture = PlainActionFuture.newFuture(); + strategy.connect(connectFuture); + IllegalStateException ise = expectThrows(IllegalStateException.class, connectFuture::actionGet); + assertThat(ise.getMessage(), allOf( + startsWith("handshake with [{other_seed}"), + containsString(otherSeed.toString()), + endsWith(" failed: remote cluster name [otherCluster] " + + "does not match expected remote cluster name [" + clusterAlias + "]"))); + + assertFalse(connectionManager.nodeConnected(otherSeedNode)); assertFalse(connectionManager.nodeConnected(otherDiscoverableNode)); assertTrue(strategy.assertNoRunningConnections()); } From cb94cd09c2980ee8636e863fdc9d413359d51272 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 27 Sep 2019 16:10:21 -0600 Subject: [PATCH 10/12] Checkstyle --- .../transport/RemoteClusterConnection.java | 20 +- .../transport/RemoteConnectionStrategy.java | 2 - .../transport/SniffConnectionStrategy.java | 4 - .../RemoteClusterConnectionTests.java | 201 ++---------------- .../SniffConnectionStrategyTests.java | 36 +++- 5 files changed, 44 insertions(+), 219 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index ccd6b770dae91..002c80019bd6b 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -55,7 +55,7 @@ * {@link RemoteClusterService#REMOTE_CONNECTIONS_PER_CLUSTER} until either all eligible nodes are exhausted or the maximum number of * connections per cluster has been reached. */ -final class RemoteClusterConnection implements TransportConnectionListener, Closeable { +final class RemoteClusterConnection implements Closeable { private final TransportService transportService; private final RemoteConnectionManager remoteConnectionManager; @@ -63,8 +63,8 @@ final class RemoteClusterConnection implements TransportConnectionListener, Clos private final String clusterAlias; private final int maxNumRemoteConnections; private final ThreadPool threadPool; - private volatile List>> seedNodes; - private volatile String proxyAddress; + private final List>> seedNodes; + private final String proxyAddress; private volatile boolean skipUnavailable; private final TimeValue initialConnectionTimeout; @@ -101,25 +101,13 @@ final class RemoteClusterConnection implements TransportConnectionListener, Clos // we register the transport service here as a listener to make sure we notify handlers on disconnect etc. connectionManager.addListener(transportService); this.seedNodes = Collections.unmodifiableList(seedNodes); + this.proxyAddress = proxyAddress; this.skipUnavailable = RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE .getConcreteSettingForNamespace(clusterAlias).get(settings); this.threadPool = transportService.threadPool; initialConnectionTimeout = RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings); } - /** - * Updates the list of seed nodes for this cluster connection - */ - synchronized void updateSeedNodes( - final String proxyAddress, - final List>> seedNodes, - final ActionListener connectListener) { - this.seedNodes = List.copyOf(seedNodes); - this.proxyAddress = proxyAddress; - // TODO: Update seedNodes - connectionStrategy.connect(connectListener); - } - /** * Updates the skipUnavailable flag that can be dynamically set for each remote cluster */ diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index abd7ac31228d7..0ff8fd2f4b0f0 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -26,12 +26,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; diff --git a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java index 9d6c381d83bf6..fc62ea2d215ba 100644 --- a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.internal.io.IOUtils; @@ -38,15 +37,12 @@ import java.io.IOException; import java.net.InetSocketAddress; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; -import static org.elasticsearch.common.settings.Setting.intSetting; - public class SniffConnectionStrategy extends RemoteConnectionStrategy { private final String clusterAlias; diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 277f36ff91a7d..6a4dab7608f27 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -78,8 +78,6 @@ import java.util.concurrent.CyclicBarrier; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; @@ -87,15 +85,11 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.hamcrest.Matchers.startsWith; -import static org.mockito.Mockito.mock; public class RemoteClusterConnectionTests extends ESTestCase { @@ -174,32 +168,6 @@ public static MockTransportService startTransport( } } - private void updateSeedNodes( - final RemoteClusterConnection connection, final List>> seedNodes) throws Exception { - updateSeedNodes(connection, seedNodes, null); - } - - private void updateSeedNodes( - final RemoteClusterConnection connection, - final List>> seedNodes, - final String proxyAddress) - throws Exception { - CountDownLatch latch = new CountDownLatch(1); - AtomicReference exceptionAtomicReference = new AtomicReference<>(); - ActionListener listener = ActionListener.wrap( - x -> latch.countDown(), - x -> { - exceptionAtomicReference.set(x); - latch.countDown(); - } - ); - connection.updateSeedNodes(proxyAddress, seedNodes, listener); - latch.await(); - if (exceptionAtomicReference.get() != null) { - throw exceptionAtomicReference.get(); - } - } - @SuppressForbidden(reason = "calls getLocalHost here but it's fine in this case") public void testSlowNodeCanBeCancelled() throws IOException, InterruptedException { try (ServerSocket socket = new MockServerSocket()) { @@ -239,7 +207,7 @@ public void run() { exceptionReference.set(x); listenerCalled.countDown(); }); - connection.updateSeedNodes(null, seedNodes(seedNode), listener); + connection.ensureConnected(listener); acceptedLatch.await(); connection.close(); // now close it, this should trigger an interrupt on the socket and we can move on assertTrue(connection.assertNoRunningConnections()); @@ -267,76 +235,6 @@ private static List>> seedNodes(final Disc } } - public void testTriggerUpdatesConcurrently() throws IOException, InterruptedException { - List knownNodes = new CopyOnWriteArrayList<>(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); - MockTransportService seedTransport1 = startTransport("seed_node_1", knownNodes, Version.CURRENT); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - DiscoveryNode seedNode1 = seedTransport1.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - knownNodes.add(seedTransport1.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - List>> seedNodes = seedNodes(seedNode1, seedNode); - Collections.shuffle(seedNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes, service, Integer.MAX_VALUE, n -> true, null, profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - int numThreads = randomIntBetween(4, 10); - Thread[] threads = new Thread[numThreads]; - CyclicBarrier barrier = new CyclicBarrier(numThreads); - for (int i = 0; i < threads.length; i++) { - final int numConnectionAttempts = randomIntBetween(10, 200); - threads[i] = new Thread() { - @Override - public void run() { - try { - barrier.await(); - CountDownLatch latch = new CountDownLatch(numConnectionAttempts); - for (int i = 0; i < numConnectionAttempts; i++) { - AtomicBoolean executed = new AtomicBoolean(false); - ActionListener listener = ActionListener.wrap( - x -> { - assertTrue(executed.compareAndSet(false, true)); - latch.countDown(); - }, - x -> { - assertTrue(executed.compareAndSet(false, true)); - latch.countDown(); - - if (!(x instanceof RejectedExecutionException)) { - throw new AssertionError(x); - } - }); - connection.updateSeedNodes(null, seedNodes, listener); - } - latch.await(); - } catch (Exception ex) { - throw new AssertionError(ex); - } - } - }; - threads[i].start(); - } - - for (int i = 0; i < threads.length; i++) { - threads[i].join(); - } - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connectionManager.nodeConnected(seedNode1)); - assertTrue(connection.assertNoRunningConnections()); - } - } - } - } - public void testCloseWhileConcurrentlyConnecting() throws IOException, InterruptedException, BrokenBarrierException { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); @@ -401,7 +299,7 @@ public void run() { } }); try { - connection.updateSeedNodes(null, seedNodes, listener); + connection.ensureConnected(listener); } catch (Exception e) { // it's ok if we're shutting down assertThat(e.getMessage(), containsString("threadcontext is already closed")); @@ -450,16 +348,6 @@ public void testGetConnectionInfo() throws Exception { assertEquals(3, remoteConnectionInfo.seedNodes.size()); assertEquals(maxNumConnections, remoteConnectionInfo.connectionsPerCluster); assertEquals("test-cluster", remoteConnectionInfo.clusterAlias); - - // Connect some nodes - updateSeedNodes(connection, seedNodes); - remoteConnectionInfo = assertSerialization(connection.getConnectionInfo()); - assertNotNull(remoteConnectionInfo); - assertEquals(connection.getNumNodesConnected(), remoteConnectionInfo.numNodesConnected); - assertEquals(Math.min(3, maxNumConnections), connection.getNumNodesConnected()); - assertEquals(3, remoteConnectionInfo.seedNodes.size()); - assertEquals(maxNumConnections, remoteConnectionInfo.connectionsPerCluster); - assertEquals("test-cluster", remoteConnectionInfo.clusterAlias); } } } @@ -543,9 +431,6 @@ public void testCollectNodes() throws Exception { service.acceptIncomingRequests(); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, profile)) { - if (randomBoolean()) { - updateSeedNodes(connection, seedNodes(seedNode)); - } CountDownLatch responseLatch = new CountDownLatch(1); AtomicReference> reference = new AtomicReference<>(); AtomicReference failReference = new AtomicReference<>(); @@ -616,25 +501,17 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted getThreads[i].start(); } - final AtomicInteger counter = new AtomicInteger(); for (int i = 0; i < modifyingThreads.length; i++) { final int numDisconnects = randomIntBetween(5, 10); modifyingThreads[i] = new Thread(() -> { try { barrier.await(); for (int j = 0; j < numDisconnects; j++) { - if (randomBoolean()) { - String node = "discoverable_node_added" + counter.incrementAndGet(); - MockTransportService transportService = - startTransport(node, knownNodes, - Version.CURRENT); - discoverableTransports.add(transportService); - seedNodes.add(Tuple.tuple(node, () -> transportService.getLocalDiscoNode())); - PlainActionFuture.get(fut -> connection.updateSeedNodes(null, seedNodes, - ActionListener.map(fut, x -> null))); - } else { - DiscoveryNode node = randomFrom(discoverableNodes).v2().get(); - connection.onNodeDisconnected(node, mock(Transport.Connection.class)); + DiscoveryNode node = randomFrom(discoverableNodes).v2().get(); + try { + connection.getConnectionManager().getConnection(node); + } catch (NodeNotConnectedException e) { + // Ignore } } } catch (Exception ex) { @@ -657,61 +534,6 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted } } - public void testClusterNameIsChecked() throws Exception { - List knownNodes = new CopyOnWriteArrayList<>(); - List otherClusterKnownNodes = new CopyOnWriteArrayList<>(); - - Settings settings = Settings.builder().put("cluster.name", "testClusterNameIsChecked").build(); - try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT, threadPool, settings); - MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, threadPool, - settings); - MockTransportService otherClusterTransport = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, - Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build()); - MockTransportService otherClusterDiscoverable = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, - Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build())) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - otherClusterKnownNodes.add(otherClusterDiscoverable.getLocalDiscoNode()); - otherClusterKnownNodes.add(otherClusterTransport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", - seedNodes(seedNode), service, Integer.MAX_VALUE, n -> true, null, profile)) { - ConnectionManager connectionManager = connection.getConnectionManager(); - updateSeedNodes(connection, seedNodes(seedNode)); - assertTrue(connectionManager.nodeConnected(seedNode)); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connection.assertNoRunningConnections()); - List>> discoveryNodes = - Arrays.asList( - Tuple.tuple("other", otherClusterTransport::getLocalDiscoNode), - Tuple.tuple(seedNode.toString(), () -> seedNode)); - Collections.shuffle(discoveryNodes, random()); - updateSeedNodes(connection, discoveryNodes); - assertTrue(connectionManager.nodeConnected(seedNode)); - for (DiscoveryNode otherClusterNode : otherClusterKnownNodes) { - assertFalse(connectionManager.nodeConnected(otherClusterNode)); - } - assertFalse(connectionManager.nodeConnected(otherClusterTransport.getLocalDiscoNode())); - assertTrue(connectionManager.nodeConnected(discoverableNode)); - assertTrue(connection.assertNoRunningConnections()); - IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, () -> - updateSeedNodes(connection, Arrays.asList(Tuple.tuple("other", otherClusterTransport::getLocalDiscoNode)))); - assertThat(illegalStateException.getMessage(), allOf( - startsWith("handshake with [{other_cluster_discoverable_node}"), - containsString(otherClusterTransport.getLocalDiscoNode().toString()), - endsWith(" failed: remote cluster name [otherCluster] " + - "does not match expected remote cluster name [testClusterNameIsChecked]"))); - } - } - } - } - public void testGetConnection() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); @@ -796,11 +618,12 @@ public void testLazyResolveTransportAddress() throws Exception { }); try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", Arrays.asList(seedSupplier), service, Integer.MAX_VALUE, n -> true, null, profile)) { - updateSeedNodes(connection, Arrays.asList(seedSupplier)); + PlainActionFuture firstConnect = PlainActionFuture.newFuture(); + connection.ensureConnected(firstConnect); + firstConnect.actionGet(); // Closing connections leads to RemoteClusterConnection.ConnectHandler.collectRemoteNodes - // being called again so we try to resolve the same seed node's host twice - discoverableTransport.close(); - seedTransport.close(); + connection.getConnectionManager().getConnection(seedNode).close(); + assertTrue(multipleResolveLatch.await(30L, TimeUnit.SECONDS)); } } diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index 75fa9680a97d3..5e901032b26fe 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -219,7 +219,7 @@ public void testConnectFailsWithIncompatibleNodes() { DiscoveryNode incompatibleSeedNode = incompatibleSeedTransport.getLocalNode(); knownNodes.add(incompatibleSeedNode); - try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { localService.start(); localService.acceptIncomingRequests(); @@ -274,18 +274,24 @@ public void testFilterNodesWithNodePredicate() { } } - public void testClusterNameIsValidated() { + public void testClusterNameValidationPreventConnectingToDifferentClusters() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); + List otherKnownNodes = new CopyOnWriteArrayList<>(); Settings otherSettings = Settings.builder().put("cluster.name", "otherCluster").build(); - try (MockTransportService otherSeed = startTransport("other_seed", knownNodes, Version.CURRENT, otherSettings); + try (MockTransportService seed = startTransport("other_seed", knownNodes, Version.CURRENT); + MockTransportService discoverable = startTransport("other_discoverable", knownNodes, Version.CURRENT); + MockTransportService otherSeed = startTransport("other_seed", knownNodes, Version.CURRENT, otherSettings); MockTransportService otherDiscoverable = startTransport("other_discoverable", knownNodes, Version.CURRENT, otherSettings)) { + DiscoveryNode seedNode = seed.getLocalNode(); + DiscoveryNode discoverableNode = discoverable.getLocalNode(); DiscoveryNode otherSeedNode = otherSeed.getLocalNode(); DiscoveryNode otherDiscoverableNode = otherDiscoverable.getLocalNode(); - knownNodes.add(otherSeedNode); - knownNodes.add(otherDiscoverableNode); + knownNodes.add(seedNode); + knownNodes.add(discoverableNode); Collections.shuffle(knownNodes, random()); + Collections.shuffle(otherKnownNodes, random()); try (MockTransportService localService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool)) { localService.start(); @@ -294,16 +300,30 @@ public void testClusterNameIsValidated() { ConnectionManager connectionManager = new ConnectionManager(profile, localService.transport); try (RemoteConnectionManager remoteConnectionManager = new RemoteConnectionManager(clusterAlias, connectionManager); SniffConnectionStrategy strategy = new SniffConnectionStrategy(clusterAlias, localService, remoteConnectionManager, - null, 3, n -> true, seedNodes(otherSeedNode))) { + null, 3, n -> true, seedNodes(seedNode, otherSeedNode))) { PlainActionFuture connectFuture = PlainActionFuture.newFuture(); strategy.connect(connectFuture); - IllegalStateException ise = expectThrows(IllegalStateException.class, connectFuture::actionGet); + connectFuture.actionGet(); + + assertTrue(connectionManager.nodeConnected(seedNode)); + assertTrue(connectionManager.nodeConnected(discoverableNode)); + assertTrue(strategy.assertNoRunningConnections()); + + seed.close(); + + assertBusy(strategy::assertNoRunningConnections); + + PlainActionFuture newConnect = PlainActionFuture.newFuture(); + strategy.connect(newConnect); + IllegalStateException ise = expectThrows(IllegalStateException.class, newConnect::actionGet); assertThat(ise.getMessage(), allOf( startsWith("handshake with [{other_seed}"), - containsString(otherSeed.toString()), + containsString(otherSeedNode.toString()), endsWith(" failed: remote cluster name [otherCluster] " + "does not match expected remote cluster name [" + clusterAlias + "]"))); + assertFalse(connectionManager.nodeConnected(seedNode)); + assertTrue(connectionManager.nodeConnected(discoverableNode)); assertFalse(connectionManager.nodeConnected(otherSeedNode)); assertFalse(connectionManager.nodeConnected(otherDiscoverableNode)); assertTrue(strategy.assertNoRunningConnections()); From a6cc4941f953718a411a406c3e2806c28c00bc50 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 30 Sep 2019 10:18:36 -0600 Subject: [PATCH 11/12] review comments --- .../transport/RemoteClusterConnection.java | 1 - .../transport/RemoteClusterService.java | 10 +++++++- .../transport/SniffConnectionStrategy.java | 1 + .../transport/RemoteClusterServiceTests.java | 24 ++++++++++++++++++- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 002c80019bd6b..4a996b0a60577 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -97,7 +97,6 @@ final class RemoteClusterConnection implements Closeable { this.connectionStrategy = new SniffConnectionStrategy(clusterAlias, transportService, remoteConnectionManager, proxyAddress, maxNumRemoteConnections, nodePredicate, Collections.unmodifiableList(seedNodes)); - connectionManager.addListener(this.connectionStrategy); // we register the transport service here as a listener to make sure we notify handlers on disconnect etc. connectionManager.addListener(transportService); this.seedNodes = Collections.unmodifiableList(seedNodes); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index 3dff3c7921abf..465ff575f8d63 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -184,7 +184,7 @@ private synchronized void updateRemoteClusters(Map>> return oldSeeds.equals(newSeeds) == false; } + private boolean proxyChanged(String oldProxy, String newProxy) { + if (oldProxy == null || oldProxy.isEmpty()) { + return (newProxy == null || newProxy.isEmpty()) == false; + } + + return Objects.equals(oldProxy, newProxy) == false; + } + /** * Collects all nodes of the given clusters and returns / passes a (clusterAlias, nodeId) to {@link DiscoveryNode} * function on success. diff --git a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java index fc62ea2d215ba..ce820b744bda8 100644 --- a/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/SniffConnectionStrategy.java @@ -78,6 +78,7 @@ protected boolean shouldOpenMoreConnections() { private void collectRemoteNodes(Iterator> seedNodes, ActionListener listener) { if (Thread.currentThread().isInterrupted()) { listener.onFailure(new InterruptedException("remote connect thread got interrupted")); + return; } if (seedNodes.hasNext()) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java index b68fd50662ea2..0de8a6bdeb00b 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java @@ -840,7 +840,7 @@ public void testGetNodePredicatesCombination() { } } - public void testReconnectWhenSeedsNodesAreUpdated() throws Exception { + public void testReconnectWhenSeedsNodesOrProxyAreUpdated() throws Exception { List knownNodes = new CopyOnWriteArrayList<>(); try (MockTransportService cluster_node_0 = startTransport("cluster_node_0", knownNodes, Version.CURRENT); MockTransportService cluster_node_1 = startTransport("cluster_node_1", knownNodes, Version.CURRENT)) { @@ -910,6 +910,28 @@ public void testReconnectWhenSeedsNodesAreUpdated() throws Exception { assertTrue(secondRemoteClusterConnection.isNodeConnected(node1)); assertEquals(2, secondRemoteClusterConnection.getNumNodesConnected()); assertFalse(secondRemoteClusterConnection.isClosed()); + + final CountDownLatch thirdLatch = new CountDownLatch(1); + service.updateRemoteCluster( + "cluster_test", + newSeeds, node1.getAddress().toString(), + genericProfile("cluster_test"), connectionListener(thirdLatch)); + thirdLatch.await(); + + assertBusy(() -> { + assertFalse(secondRemoteClusterConnection.isNodeConnected(node0)); + assertFalse(secondRemoteClusterConnection.isNodeConnected(node1)); + assertEquals(0, secondRemoteClusterConnection.getNumNodesConnected()); + assertTrue(secondRemoteClusterConnection.isClosed()); + }); + + final RemoteClusterConnection thirdRemoteClusterConnection = service.getRemoteClusterConnection("cluster_test"); + assertTrue(thirdRemoteClusterConnection.isNodeConnected(node1)); + // Will only successfully connect to node1 because the proxy address is to node1 and + // validation will fail when attempt to connect to node0 + assertFalse(thirdRemoteClusterConnection.isNodeConnected(node0)); + assertEquals(1, thirdRemoteClusterConnection.getNumNodesConnected()); + assertFalse(thirdRemoteClusterConnection.isClosed()); } } } From ca40fd57e3c30cb9b175bc8e2d51ca5036296cde Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 30 Sep 2019 10:49:18 -0600 Subject: [PATCH 12/12] Remove listener in strategy --- .../org/elasticsearch/transport/RemoteClusterConnection.java | 1 - .../org/elasticsearch/transport/RemoteConnectionStrategy.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 4a996b0a60577..dd3cd5ea07a5a 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -208,7 +208,6 @@ Transport.Connection getConnection() { @Override public void close() throws IOException { - remoteConnectionManager.getConnectionManager().removeListener(connectionStrategy); IOUtils.close(connectionStrategy, remoteConnectionManager); } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index 0ff8fd2f4b0f0..191484deff2f5 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -123,6 +123,7 @@ public void close() { final List> toNotify; synchronized (mutex) { if (closed.compareAndSet(false, true)) { + connectionManager.getConnectionManager().removeListener(this); toNotify = listeners; listeners = Collections.emptyList(); } else {