Bind to non-localhost for transport in some cases (#82973) (#83096)

When enrolling a new node to an existing cluster, we sometimes need to
bind transport to non-localhost addresse. If the other nodes of the
cluster are on different hosts than this node, then the default
configuration of binding transport layer to localhost will prevent this
node to join the cluster even after "successful" enrollment.
We check the non-localhost transport addresses that we receive during
enrollment and if any of these are not in the list of non-localhost IP
addresses that we gather from all interfaces of the current host, we
assume that at least some other node in the cluster runs on another host.

docs/changelog/82973.yaml

@@ -0,0 +1,5 @@
+pr: 82973
+summary: Bind to non-localhost for transport in some cases
+area: Security
+type: bug
+issues: []

x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/AutoConfigureNode.java

@@ -54,7 +54,10 @@
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
@@ -799,7 +802,11 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th
                         bw.newLine();
                         bw.write("# Connections are encrypted and mutually authenticated");
                         bw.newLine();
-                        bw.write("#" + TransportSettings.HOST.getKey() + ": " + hostSettingValue(NetworkUtils.getAllAddresses()));
+                        if (false == inEnrollmentMode
+                            || false == anyRemoteHostNodeAddress(transportAddresses, NetworkUtils.getAllAddresses())) {
+                            bw.write("#");
+                        }
+                        bw.write(TransportSettings.HOST.getKey() + ": " + hostSettingValue(NetworkUtils.getAllAddresses()));
                         bw.newLine();
                     }
                     bw.newLine();
@@ -846,6 +853,33 @@ private String initialMasterNodesSettingValue(Environment environment) {
         return "[\"${HOSTNAME}\"]";
     }
 
+    /**
+     * Determines if a node that is enrolling to an existing cluster is on a different host than the other nodes of the
+     * cluster. If this is the case, then the default configuration of
+     * binding transport layer to localhost will prevent this node to join the cluster even after "successful" enrollment.
+     * We check the non-localhost transport addresses that we receive during enrollment and if any of these are not in the
+     * list of non-localhost IP addresses that we gather from all interfaces of the current host, we assume that at least
+     * some other node in the cluster runs on another host.
+     * If the transport layer addresses we found out in enrollment are all localhost, we cannot be sure where we are still
+     * on the same host, but we assume that as it is safer to do so and do not bind to non localhost for this node either.
+     */
+    protected static boolean anyRemoteHostNodeAddress(List<String> allNodesTransportPublishAddresses, InetAddress[] allHostAddresses) {
+        final List<InetAddress> allAddressesList = Arrays.asList(allHostAddresses);
+        for (String nodeStringAddress : allNodesTransportPublishAddresses) {
+            try {
+                final URI uri = new URI("http://" + nodeStringAddress);
+                final InetAddress nodeAddress = InetAddress.getByName(uri.getHost());
+                if (false == nodeAddress.isLoopbackAddress() && false == allAddressesList.contains(nodeAddress)) {
+                    // this node's address is on a remote host
+                    return true;
+                }
+            } catch (URISyntaxException | UnknownHostException e) {
+                // we could fail here but if any of the transport addresses are usable, we can join the cluster
+            }
+        }
+        return false;
+    }
+
     protected String hostSettingValue(InetAddress[] allAddresses) {
         if (Arrays.stream(allAddresses).anyMatch(InetAddress::isSiteLocalAddress)) {
             return "[_local_, _site_]";

x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/AutoConfigureNodeTests.java

@@ -24,14 +24,17 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyStore;
 import java.security.cert.X509Certificate;
 import java.util.List;
 
 import static java.nio.file.StandardOpenOption.CREATE_NEW;
+import static org.elasticsearch.xpack.security.cli.AutoConfigureNode.anyRemoteHostNodeAddress;
 import static org.elasticsearch.xpack.security.cli.AutoConfigureNode.removePreviousAutoconfiguration;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class AutoConfigureNodeTests extends ESTestCase {
@@ -210,6 +213,43 @@ public void testGeneratedHTTPCertificateSANs() throws Exception {
         }
     }
 
+    public void testAnyRemoteHostNodeAddress() throws Exception {
+        List<String> remoteAddresses = List.of("192.168.0.1:9300", "127.0.0.1:9300");
+        InetAddress[] localAddresses = new InetAddress[] { InetAddress.getByName("192.168.0.1"), InetAddress.getByName("127.0.0.1") };
+        assertThat(anyRemoteHostNodeAddress(remoteAddresses, localAddresses), equalTo(false));
+
+        remoteAddresses = List.of("192.168.0.1:9300", "127.0.0.1:9300", "[::1]:9300");
+        localAddresses = new InetAddress[] { InetAddress.getByName("192.168.0.1"), InetAddress.getByName("127.0.0.1") };
+        assertThat(anyRemoteHostNodeAddress(remoteAddresses, localAddresses), equalTo(false));
+
+        remoteAddresses = List.of("192.168.0.1:9300", "127.0.0.1:9300", "[::1]:9300");
+        localAddresses = new InetAddress[] {
+            InetAddress.getByName("192.168.0.1"),
+            InetAddress.getByName("127.0.0.1"),
+            InetAddress.getByName("10.0.0.1") };
+        assertThat(anyRemoteHostNodeAddress(remoteAddresses, localAddresses), equalTo(false));
+
+        remoteAddresses = List.of("192.168.0.1:9300", "127.0.0.1:9300", "[::1]:9300", "10.0.0.1:9301");
+        localAddresses = new InetAddress[] { InetAddress.getByName("192.168.0.1"), InetAddress.getByName("127.0.0.1") };
+        assertThat(anyRemoteHostNodeAddress(remoteAddresses, localAddresses), equalTo(true));
+
+        remoteAddresses = List.of("127.0.0.1:9300", "[::1]:9300");
+        localAddresses = new InetAddress[] { InetAddress.getByName("[::1]"), InetAddress.getByName("127.0.0.1") };
+        assertThat(anyRemoteHostNodeAddress(remoteAddresses, localAddresses), equalTo(false));
+
+        remoteAddresses = List.of("127.0.0.1:9300", "[::1]:9300");
+        localAddresses = new InetAddress[] { InetAddress.getByName("192.168.2.3") };
+        assertThat(anyRemoteHostNodeAddress(remoteAddresses, localAddresses), equalTo(false));
+
+        remoteAddresses = List.of("1.2.3.4:9300");
+        localAddresses = new InetAddress[] { InetAddress.getByName("[::1]"), InetAddress.getByName("127.0.0.1") };
+        assertThat(anyRemoteHostNodeAddress(remoteAddresses, localAddresses), equalTo(true));
+
+        remoteAddresses = List.of();
+        localAddresses = new InetAddress[] { InetAddress.getByName("192.168.0.1"), InetAddress.getByName("127.0.0.1") };
+        assertThat(anyRemoteHostNodeAddress(remoteAddresses, localAddresses), equalTo(false));
+    }
+
     private boolean checkGeneralNameSan(X509Certificate certificate, String generalName, int generalNameTag) throws Exception {
         for (List<?> san : certificate.getSubjectAlternativeNames()) {
             if (san.get(0).equals(generalNameTag) && san.get(1).equals(generalName)) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java

@@ -254,7 +254,7 @@ private static void outputInformationToConsole(
         } else if (false == Strings.isEmpty(elasticPassword)) {
             builder.append(
                 infoBullet
-                    + " Password for the "
+                    + "  Password for the "
                     + boldOnANSI
                     + "elastic"
                     + boldOffANSI
@@ -271,15 +271,15 @@ private static void outputInformationToConsole(
         builder.append(System.lineSeparator());
 
         if (null != caCertFingerprint) {
-            builder.append(infoBullet + " HTTP CA certificate SHA-256 fingerprint:");
+            builder.append(infoBullet + "  HTTP CA certificate SHA-256 fingerprint:");
             builder.append(System.lineSeparator());
             builder.append("  " + boldOnANSI + caCertFingerprint + boldOffANSI);
         }
         builder.append(System.lineSeparator());
         builder.append(System.lineSeparator());
 
         if (null != kibanaEnrollmentToken) {
-            builder.append(infoBullet + " Configure Kibana to use this cluster:");
+            builder.append(infoBullet + "  Configure Kibana to use this cluster:");
             builder.append(System.lineSeparator());
             builder.append(bullet + " Run Kibana and click the configuration link in the terminal when Kibana starts.");
             builder.append(System.lineSeparator());
@@ -325,7 +325,7 @@ private static void outputInformationToConsole(
                     + ", using the enrollment token that you generated."
             );
         } else if (Strings.isEmpty(nodeEnrollmentToken)) {
-            builder.append(infoBullet + " Configure other nodes to join this cluster:");
+            builder.append(infoBullet + "  Configure other nodes to join this cluster:");
             builder.append(System.lineSeparator());
             builder.append(bullet + " On this node:");
             builder.append(System.lineSeparator());