Change skip_unavailable default value to true

docs/changelog/105792.yaml

@@ -0,0 +1,18 @@
+pr: 105792
+summary: "Change `skip_unavailable` remote cluster setting default value to true"
+area: Search
+type: breaking
+issues: []
+breaking:
+  title: "Change `skip_unavailable` remote cluster setting default value to true"
+  area: Cluster and node setting
+  details: The default value of the `skip_unavailable` setting is now set to true.
+    All existing and future remote clusters that do not define this setting will use the new default.
+    This setting only affects cross-cluster searches using the _search or _async_search API.
+  impact: Unavailable remote clusters in a cross-cluster search will no longer cause the search to fail unless
+    skip_unavailable is configured to be `false` in elasticsearch.yml or via the `_cluster/settings` API. 
+    Unavailable clusters with `skip_unavailable`=`true` (either explicitly or by using the new default) are marked 
+    as SKIPPED in the search response metadata section and do not fail the entire search. If users want to ensure that a
+    search returns a failure when a particular remote cluster is not available, `skip_unavailable` must be now be 
+    set explicitly.    
+  notable: false

docs/reference/ccr/getting-started.asciidoc

@@ -147,7 +147,7 @@ cluster with cluster alias `leader`.
     "num_nodes_connected" : 1, <1>
     "max_connections_per_cluster" : 3,
     "initial_connect_timeout" : "30s",
-    "skip_unavailable" : false,
+    "skip_unavailable" : true,
     "mode" : "sniff"
   }
 }

docs/reference/modules/cluster/remote-clusters-connect.asciidoc

@@ -37,7 +37,7 @@ clusters on individual nodes in the local cluster, define static settings in
 `elasticsearch.yml` for each node.
 
 The following request adds a remote cluster with an alias of `cluster_one`. This
-_cluster alias_ is a unique identifier that represents the connection to the 
+_cluster alias_ is a unique identifier that represents the connection to the
 remote cluster and is used to distinguish between local and remote indices.
 
 [source,console,subs=attributes+]
@@ -60,7 +60,7 @@ PUT /_cluster/settings
 // TEST[setup:host]
 // TEST[s/127.0.0.1:\{remote-interface-default-port\}/\${transport_host}/]
 <1> The cluster alias of this remote cluster is `cluster_one`.
-<2> Specifies the hostname and {remote-interface} port of a seed node in the 
+<2> Specifies the hostname and {remote-interface} port of a seed node in the
 remote cluster.
 
 You can use the <<cluster-remote-info,remote cluster info API>> to verify that
@@ -86,7 +86,7 @@ cluster with the cluster alias `cluster_one`:
     "num_nodes_connected" : 1,  <1>
     "max_connections_per_cluster" : 3,
     "initial_connect_timeout" : "30s",
-    "skip_unavailable" : false, <2>
+    "skip_unavailable" : true, <2>
 ifeval::["{trust-mechanism}"=="api-key"]
     "cluster_credentials": "::es_redacted::", <3>
 endif::[]
@@ -103,7 +103,7 @@ connected to.
 <2> Indicates whether to skip the remote cluster if searched through {ccs} but
 no nodes are available.
 ifeval::["{trust-mechanism}"=="api-key"]
-<3> If present, indicates the remote cluster has connected using API key 
+<3> If present, indicates the remote cluster has connected using API key
 authentication.
 endif::[]
 
@@ -187,7 +187,7 @@ PUT _cluster/settings
 
 You can delete a remote cluster from the cluster settings by passing `null`
 values for each remote cluster setting. The following request removes
-`cluster_two` from the cluster settings, leaving `cluster_one` and 
+`cluster_two` from the cluster settings, leaving `cluster_one` and
 `cluster_three` intact:
 
 [source,console]
@@ -212,15 +212,15 @@ PUT _cluster/settings
 
 ===== Statically configure remote clusters
 If you specify settings in `elasticsearch.yml`, only the nodes with
-those settings can connect to the remote cluster and serve remote cluster 
+those settings can connect to the remote cluster and serve remote cluster
 requests.
 
-NOTE: Remote cluster settings that are specified using the 
+NOTE: Remote cluster settings that are specified using the
 <<cluster-update-settings,cluster update settings API>> take precedence over
 settings that you specify in `elasticsearch.yml` for individual nodes.
 
-In the following example, `cluster_one`, `cluster_two`, and `cluster_three` are 
-arbitrary cluster aliases representing the connection to each cluster. These 
+In the following example, `cluster_one`, `cluster_two`, and `cluster_three` are
+arbitrary cluster aliases representing the connection to each cluster. These
 names are subsequently used to distinguish between local and remote indices.
 
 [source,yaml,subs=attributes+]

docs/reference/modules/cluster/remote-clusters-settings.asciidoc

@@ -28,9 +28,14 @@ mode are described separately.
 
   Per cluster boolean setting that allows to skip specific clusters when no
   nodes belonging to them are available and they are the target of a remote
-  cluster request. Default is `false`, meaning that all clusters are mandatory
-  by default, but they can selectively be made optional by setting this setting
-  to `true`.
+  cluster request.
+
+IMPORTANT: In Elasticsearch 8.15, the default value for `skip_unavailable` was
+changed from `false` to `true`. Before Elasticsearch 8.15, if you want a cluster
+to be treated as optional for a {ccs}, then you need to set that configuration.
+From Elasticsearch 8.15 forward, you need to set the configuration in order to
+make a cluster required for the {ccs}.
+
 
 `cluster.remote.<cluster_alias>.transport.ping_schedule`::
 

docs/reference/search/search-your-data/search-across-clusters.asciidoc

@@ -1178,7 +1178,13 @@ gathered from all 3 clusters and the total shard count on each cluster is listed
 By default, a {ccs} fails if a remote cluster in the request is unavailable
 or returns an error where the search on all shards failed. Use the
 `skip_unavailable` cluster setting to mark a specific remote cluster as
-optional for {ccs}.
+either optional or required for {ccs}.
+
+IMPORTANT: In Elasticsearch 8.15, the default value for `skip_unavailable` was
+changed from `false` to `true`. Before Elasticsearch 8.15, if you want a cluster
+to be treated as optional for a {ccs}, then you need to set that configuration.
+From Elasticsearch 8.15 forward, you need to set the configuration in order to
+make a cluster required for the {ccs}.
 
 If `skip_unavailable` is `true`, a {ccs}:
 
@@ -1196,25 +1202,33 @@ parameter and the related `search.default_allow_partial_results` cluster setting
 when searching the remote cluster. This means searches on the remote cluster may
 return partial results.
 
-The following <<cluster-update-settings,cluster update settings>>
-API request changes `skip_unavailable` setting to `true` for `cluster_two`.
-
-[source,console]
---------------------------------
-PUT _cluster/settings
-{
-  "persistent": {
-    "cluster.remote.cluster_two.skip_unavailable": true
-  }
-}
---------------------------------
-// TEST[continued]
-
-If `cluster_two` is disconnected or unavailable during a {ccs}, {es} won't
-include matching documents from that cluster in the final results. If at
-least one shard provides results, those results will be used and the
-search will return partial data. (If doing {ccs} using async search,
-the `is_partial` field will be set to `true` to indicate partial results.)
+You can modify the `skip_unavailable` setting by editing the `cluster.remote.<cluster_alias>`
+settings in the elasticsearch.yml config file. For example:
+
+```
+cluster:
+    remote:
+        cluster_one:
+            seeds: 35.238.149.1:9300
+            skip_unavailable: false
+        cluster_two:
+            seeds: 35.238.149.2:9300
+            skip_unavailable: true
+```
+
+Or you can set the cluster.remote settings via the
+<<cluster-update-settings,cluster update settings>> API as shown
+<<ccs-remote-cluster-setup, here>>.
+
+When a remote cluster configured with `skip_unavailable: true` (such as
+`cluster_two` above) is disconnected or unavailable during a {ccs}, {es} won't
+include matching documents from that cluster in the final results and the
+search will be considered successful (HTTP status 200 OK).
+
+If at least one shard from a cluster provides search results, those results will
+be used and the search will return partial data. This is true regardless of
+the `skip_unavailable` setting of the remote cluster. (If doing {ccs} using async
+search, the `is_partial` field will be set to `true` to indicate partial results.)
 
 [discrete]
 [[ccs-network-delays]]

modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java

@@ -19,6 +19,7 @@
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
@@ -38,6 +39,11 @@ protected Collection<String> remoteClusterAlias() {
         return List.of(REMOTE_CLUSTER);
     }
 
+    @Override
+    protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
+        return Map.of(REMOTE_CLUSTER, false);
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
         return List.of(ReindexPlugin.class);

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java

@@ -101,6 +101,7 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .setting("node.roles", "[data,ingest,master,remote_cluster_client]")
         .setting("cluster.remote.remote_cluster.seeds", () -> "\"" + remoteCluster.getTransportEndpoint(0) + "\"")
         .setting("cluster.remote.connections_per_cluster", "1")
+        .setting("cluster.remote.remote_cluster.skip_unavailable", "false")
         .apply(commonClusterConfig)
         .build();
 

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java

@@ -246,6 +246,7 @@ public void initSearchClient() throws IOException {
 
     private static void configureRemoteCluster() throws IOException {
         final Settings.Builder builder = Settings.builder();
+        builder.put("cluster.remote." + REMOTE_CLUSTER_NAME + ".skip_unavailable", "false");
         if (randomBoolean()) {
             builder.put("cluster.remote." + REMOTE_CLUSTER_NAME + ".mode", "proxy")
                 .put("cluster.remote." + REMOTE_CLUSTER_NAME + ".proxy_address", fulfillingCluster.getRemoteClusterServerEndpoint(0));

qa/multi-cluster-search/build.gradle

@@ -48,6 +48,7 @@ BuildParams.bwcVersions.withWireCompatible(ccsSupportedVersion) { bwcVersion, ba
     setting 'cluster.remote.connections_per_cluster', '1'
     setting 'cluster.remote.my_remote_cluster.seeds',
       { "\"${remoteCluster.get().getAllTransportPortURI().get(0)}\"" }
+    setting 'cluster.remote.my_remote_cluster.skip_unavailable', 'false'
   }
 
   tasks.register("${baseName}#remote-cluster", RestIntegTestTask) {

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml

@@ -249,7 +249,7 @@
           persistent:
             cluster.remote.test_remote_cluster.seeds: $remote_ip
 
-  - match: {persistent: {cluster.remote.test_remote_cluster.seeds: $remote_ip}}
+  - match: {persistent.cluster\.remote\.test_remote_cluster\.seeds: $remote_ip}
 
   - do:
       search:

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/20_info.yml

@@ -113,33 +113,33 @@
 
   - do:
       cluster.remote_info: {}
-  - is_false: remote1.skip_unavailable
+  - is_true: remote1.skip_unavailable
 
   - do:
       cluster.put_settings:
         body:
           persistent:
-            cluster.remote.remote1.skip_unavailable: true
+            cluster.remote.remote1.skip_unavailable: false
 
-  - is_true: persistent.cluster.remote.remote1.skip_unavailable
+  - is_false: persistent.cluster.remote.remote1.skip_unavailable
 
   - do:
       cluster.remote_info: {}
 
-  - is_true: remote1.skip_unavailable
+  - is_false: remote1.skip_unavailable
 
   - do:
       cluster.put_settings:
         body:
           persistent:
-            cluster.remote.remote1.skip_unavailable: false
+            cluster.remote.remote1.skip_unavailable: true
 
-  - is_false: persistent.cluster.remote.remote1.skip_unavailable
+  - is_true: persistent.cluster.remote.remote1.skip_unavailable
 
   - do:
       cluster.remote_info: {}
 
-  - is_false: remote1.skip_unavailable
+  - is_true: remote1.skip_unavailable
 
   - do:
       cluster.put_settings:
@@ -152,7 +152,7 @@
   - do:
       cluster.remote_info: {}
 
-  - is_false: remote1.skip_unavailable
+  - is_true: remote1.skip_unavailable
 
   - do:
       cluster.put_settings:

server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java

@@ -87,7 +87,7 @@ public final class RemoteClusterService extends RemoteClusterAware implements Cl
     public static final Setting.AffixSetting<Boolean> REMOTE_CLUSTER_SKIP_UNAVAILABLE = Setting.affixKeySetting(
         "cluster.remote.",
         "skip_unavailable",
-        (ns, key) -> boolSetting(key, false, new RemoteConnectionEnabled<>(ns, key), Setting.Property.Dynamic, Setting.Property.NodeScope)
+        (ns, key) -> boolSetting(key, true, new RemoteConnectionEnabled<>(ns, key), Setting.Property.Dynamic, Setting.Property.NodeScope)
     );
 
     public static final Setting.AffixSetting<TimeValue> REMOTE_CLUSTER_PING_SCHEDULE = Setting.affixKeySetting(

server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java

@@ -469,7 +469,8 @@ private MockTransportService[] startTransport(
         int numClusters,
         DiscoveryNode[] nodes,
         Map<String, OriginalIndices> remoteIndices,
-        Settings.Builder settingsBuilder
+        Settings.Builder settingsBuilder,
+        boolean skipUnavailable
     ) {
         MockTransportService[] mockTransportServices = new MockTransportService[numClusters];
         for (int i = 0; i < numClusters; i++) {
@@ -486,6 +487,7 @@ private MockTransportService[] startTransport(
             knownNodes.add(remoteSeedNode);
             nodes[i] = remoteSeedNode;
             settingsBuilder.put("cluster.remote.remote" + i + ".seeds", remoteSeedNode.getAddress().toString());
+            settingsBuilder.put("cluster.remote.remote" + i + ".skip_unavailable", Boolean.toString(skipUnavailable));
             remoteIndices.put("remote" + i, new OriginalIndices(new String[] { "index" }, IndicesOptions.lenientExpandOpen()));
         }
         return mockTransportServices;
@@ -496,7 +498,8 @@ public void testCCSRemoteReduceMergeFails() throws Exception {
         DiscoveryNode[] nodes = new DiscoveryNode[numClusters];
         Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
         Settings.Builder builder = Settings.builder();
-        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder);
+        boolean skipUnavailable = randomBoolean();
+        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder, skipUnavailable);
         Settings settings = builder.build();
         boolean local = randomBoolean();
         OriginalIndices localIndices = local ? new OriginalIndices(new String[] { "index" }, SearchRequest.DEFAULT_INDICES_OPTIONS) : null;
@@ -566,7 +569,8 @@ public void testCCSRemoteReduce() throws Exception {
         DiscoveryNode[] nodes = new DiscoveryNode[numClusters];
         Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
         Settings.Builder builder = Settings.builder();
-        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder);
+        boolean skipUnavailable = randomBoolean();
+        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder, skipUnavailable);
         Settings settings = builder.build();
         boolean local = randomBoolean();
         OriginalIndices localIndices = local ? new OriginalIndices(new String[] { "index" }, SearchRequest.DEFAULT_INDICES_OPTIONS) : null;
@@ -709,7 +713,8 @@ public void testCCSRemoteReduceWhereRemoteClustersFail() throws Exception {
         DiscoveryNode[] nodes = new DiscoveryNode[numClusters];
         Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
         Settings.Builder builder = Settings.builder();
-        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder);
+        boolean skipUnavailable = randomBoolean();
+        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder, skipUnavailable);
         Settings settings = builder.build();
         boolean local = randomBoolean();
         OriginalIndices localIndices = local ? new OriginalIndices(new String[] { "index" }, SearchRequest.DEFAULT_INDICES_OPTIONS) : null;
@@ -734,10 +739,13 @@ public void testCCSRemoteReduceWhereRemoteClustersFail() throws Exception {
                 final CountDownLatch latch = new CountDownLatch(1);
                 SetOnce<Tuple<SearchRequest, ActionListener<SearchResponse>>> setOnce = new SetOnce<>();
                 AtomicReference<Exception> failure = new AtomicReference<>();
-                LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(
-                    ActionListener.wrap(r -> fail("no response expected"), failure::set),
-                    latch
-                );
+                LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(ActionListener.wrap(r -> {
+                    if (skipUnavailable) {
+                        assertThat(r.getClusters().getClusterStateCount(SearchResponse.Cluster.Status.SKIPPED), equalTo(numClusters));
+                    } else {
+                        fail("no response expected");  // failure should be returned, not SearchResponse
+                    }
+                }, failure::set), latch);
 
                 TaskId parentTaskId = new TaskId("n", 1);
                 SearchTask task = new SearchTask(2, "search", "search", () -> "desc", parentTaskId, Collections.emptyMap());
@@ -763,10 +771,14 @@ public void testCCSRemoteReduceWhereRemoteClustersFail() throws Exception {
                     resolveWithEmptySearchResponse(tuple);
                 }
                 awaitLatch(latch, 5, TimeUnit.SECONDS);
-                assertNotNull(failure.get());
-                assertThat(failure.get(), instanceOf(RemoteTransportException.class));
-                RemoteTransportException remoteTransportException = (RemoteTransportException) failure.get();
-                assertEquals(RestStatus.NOT_FOUND, remoteTransportException.status());
+                if (skipUnavailable) {
+                    assertNull(failure.get());
+                } else {
+                    assertNotNull(failure.get());
+                    assertThat(failure.get(), instanceOf(RemoteTransportException.class));
+                    RemoteTransportException remoteTransportException = (RemoteTransportException) failure.get();
+                    assertEquals(RestStatus.NOT_FOUND, remoteTransportException.status());
+                }
             }
 
         } finally {
@@ -781,7 +793,7 @@ public void testCCSRemoteReduceWithDisconnectedRemoteClusters() throws Exception
         DiscoveryNode[] nodes = new DiscoveryNode[numClusters];
         Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
         Settings.Builder builder = Settings.builder();
-        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder);
+        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder, false);
         Settings settings = builder.build();
         boolean local = randomBoolean();
         OriginalIndices localIndices = local ? new OriginalIndices(new String[] { "index" }, SearchRequest.DEFAULT_INDICES_OPTIONS) : null;
@@ -1035,7 +1047,7 @@ public void testCollectSearchShards() throws Exception {
         DiscoveryNode[] nodes = new DiscoveryNode[numClusters];
         Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
         Settings.Builder builder = Settings.builder();
-        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder);
+        MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder, false);
         Settings settings = builder.build();
         try (
             MockTransportService service = MockTransportService.createNewService(