Add support for index pattern exclusion in CCR AutoFollow

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java

@@ -82,11 +82,19 @@ public static class Pattern extends FollowConfig {
 
         @SuppressWarnings("unchecked")
         private static final ConstructingObjectParser<Pattern, Void> PARSER = new ConstructingObjectParser<>(
-            "pattern", true, args -> new Pattern((String) args[0], (List<String>) args[1], (String) args[2]));
+            "pattern",
+            true,
+            args -> new Pattern((String) args[0],
+                                (List<String>) args[1],
+                                args[2] == null ? Collections.emptyList() : (List<String>) args[2],
+                                (String) args[3])
+        );
 
         static {
             PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);
             PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD);
+            PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(),
+                                      PutAutoFollowPatternRequest.LEADER_EXCLUSION_PATTERNS_FIELD);
             PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD);
             PARSER.declareObject(Pattern::setSettings, (p, c) -> Settings.fromXContent(p), PutAutoFollowPatternRequest.SETTINGS);
             PARSER.declareInt(Pattern::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT);
@@ -123,11 +131,16 @@ public static class Pattern extends FollowConfig {
 
         private final String remoteCluster;
         private final List<String> leaderIndexPatterns;
+        private final List<String> leaderIndexExclusionPatterns;
         private final String followIndexNamePattern;
 
-        Pattern(String remoteCluster, List<String> leaderIndexPatterns, String followIndexNamePattern) {
+        Pattern(String remoteCluster,
+                List<String> leaderIndexPatterns,
+                List<String> leaderIndexExclusionPatterns,
+                String followIndexNamePattern) {
             this.remoteCluster = remoteCluster;
             this.leaderIndexPatterns = leaderIndexPatterns;
+            this.leaderIndexExclusionPatterns = leaderIndexExclusionPatterns;
             this.followIndexNamePattern = followIndexNamePattern;
         }
 
@@ -139,6 +152,10 @@ public List<String> getLeaderIndexPatterns() {
             return leaderIndexPatterns;
         }
 
+        public List<String> getLeaderIndexExclusionPatterns() {
+            return leaderIndexExclusionPatterns;
+        }
+
         public String getFollowIndexNamePattern() {
             return followIndexNamePattern;
         }
@@ -151,6 +168,7 @@ public boolean equals(Object o) {
             Pattern pattern = (Pattern) o;
             return Objects.equals(remoteCluster, pattern.remoteCluster) &&
                 Objects.equals(leaderIndexPatterns, pattern.leaderIndexPatterns) &&
+                Objects.equals(leaderIndexExclusionPatterns, pattern.leaderIndexExclusionPatterns) &&
                 Objects.equals(followIndexNamePattern, pattern.followIndexNamePattern);
         }
 
@@ -160,6 +178,7 @@ public int hashCode() {
                 super.hashCode(),
                 remoteCluster,
                 leaderIndexPatterns,
+                leaderIndexExclusionPatterns,
                 followIndexNamePattern
             );
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java

@@ -14,23 +14,36 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
 public final class PutAutoFollowPatternRequest extends FollowConfig implements Validatable, ToXContentObject {
 
     static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns");
+    static final ParseField LEADER_EXCLUSION_PATTERNS_FIELD = new ParseField("leader_index_exclusion_patterns");
     static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern");
 
     private final String name;
     private final String remoteCluster;
     private final List<String> leaderIndexPatterns;
+    private final List<String> leaderIndexExclusionPatterns;
     private String followIndexNamePattern;
 
-    public PutAutoFollowPatternRequest(String name, String remoteCluster, List<String> leaderIndexPatterns) {
+    public PutAutoFollowPatternRequest(String name,
+                                       String remoteCluster,
+                                       List<String> leaderIndexPatterns) {
+        this(name, remoteCluster, leaderIndexPatterns, Collections.emptyList());
+    }
+
+    public PutAutoFollowPatternRequest(String name,
+                                       String remoteCluster,
+                                       List<String> leaderIndexPatterns,
+                                       List<String> leaderIndexExclusionPatterns) {
         this.name = Objects.requireNonNull(name);
         this.remoteCluster = Objects.requireNonNull(remoteCluster);
         this.leaderIndexPatterns = Objects.requireNonNull(leaderIndexPatterns);
+        this.leaderIndexExclusionPatterns = Objects.requireNonNull(leaderIndexExclusionPatterns);
     }
 
     public String getName() {
@@ -45,6 +58,10 @@ public List<String> getLeaderIndexPatterns() {
         return leaderIndexPatterns;
     }
 
+    public List<String> getLeaderIndexExclusionPatterns() {
+        return leaderIndexExclusionPatterns;
+    }
+
     public String getFollowIndexNamePattern() {
         return followIndexNamePattern;
     }
@@ -58,6 +75,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(PutFollowRequest.REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster);
         builder.field(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns);
+        builder.field(LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns);
         if (followIndexNamePattern != null) {
             builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern);
         }
@@ -75,6 +93,7 @@ public boolean equals(Object o) {
         return Objects.equals(name, that.name) &&
             Objects.equals(remoteCluster, that.remoteCluster) &&
             Objects.equals(leaderIndexPatterns, that.leaderIndexPatterns) &&
+            Objects.equals(leaderIndexExclusionPatterns, that.leaderIndexExclusionPatterns) &&
             Objects.equals(followIndexNamePattern, that.followIndexNamePattern);
     }
 
@@ -85,6 +104,7 @@ public int hashCode() {
             name,
             remoteCluster,
             leaderIndexPatterns,
+            leaderIndexExclusionPatterns,
             followIndexNamePattern
         );
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java

@@ -244,8 +244,10 @@ public void testForgetFollower() throws IOException {
 
     public void testAutoFollowing() throws Exception {
         CcrClient ccrClient = highLevelClient().ccr();
-        PutAutoFollowPatternRequest putAutoFollowPatternRequest =
-            new PutAutoFollowPatternRequest("pattern1", "local_cluster", Collections.singletonList("logs-*"));
+        PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("pattern1",
+                "local_cluster",
+                Collections.singletonList("logs-*"),
+                Collections.singletonList("logs-excluded"));
         putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}");
         final int followerNumberOfReplicas = randomIntBetween(0, 4);
         final Settings autoFollowerPatternSettings =
@@ -270,6 +272,20 @@ public void testAutoFollowing() throws Exception {
             getIndexSettingsAsMap("copy-logs-20200101"),
             hasEntry("index.number_of_replicas", Integer.toString(followerNumberOfReplicas)));
 
+        CreateIndexRequest createExcludedIndexRequest = new CreateIndexRequest("logs-excluded");
+        CreateIndexResponse createExcludedIndexResponse =
+            highLevelClient().indices().create(createExcludedIndexRequest, RequestOptions.DEFAULT);
+        assertThat(createExcludedIndexResponse.isAcknowledged(), is(true));
+
+        assertBusy(() -> {
+            CcrStatsRequest ccrStatsRequest = new CcrStatsRequest();
+            CcrStatsResponse ccrStatsResponse = execute(ccrStatsRequest, ccrClient::getCcrStats, ccrClient::getCcrStatsAsync);
+            assertThat(ccrStatsResponse.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(1L));
+            assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-20200101"), notNullValue());
+        });
+
+        assertThat(indexExists("copy-logs-excluded"), is(false));
+
         GetAutoFollowPatternRequest getAutoFollowPatternRequest =
             randomBoolean() ? new GetAutoFollowPatternRequest("pattern1") : new GetAutoFollowPatternRequest();
         GetAutoFollowPatternResponse getAutoFollowPatternResponse =
@@ -279,6 +295,7 @@ public void testAutoFollowing() throws Exception {
         assertThat(pattern, notNullValue());
         assertThat(pattern.getRemoteCluster(), equalTo(putAutoFollowPatternRequest.getRemoteCluster()));
         assertThat(pattern.getLeaderIndexPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexPatterns()));
+        assertThat(pattern.getLeaderIndexExclusionPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexExclusionPatterns()));
         assertThat(pattern.getFollowIndexNamePattern(), equalTo(putAutoFollowPatternRequest.getFollowIndexNamePattern()));
         assertThat(pattern.getSettings(), equalTo(autoFollowerPatternSettings));
 

client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java

@@ -101,7 +101,10 @@ public void testForgetFollower() throws IOException {
 
     public void testPutAutofollowPattern() throws Exception {
         PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest(randomAlphaOfLength(4),
-            randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
+            randomAlphaOfLength(4),
+            Arrays.asList(generateRandomStringArray(4, 4, false)),
+            Arrays.asList(generateRandomStringArray(4, 4, false))
+        );
         if (randomBoolean()) {
             putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
         }

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java

@@ -39,6 +39,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
         for (int i = 0; i < numPatterns; i++) {
             String remoteCluster = randomAlphaOfLength(4);
             List<String> leaderIndexPatterns = Collections.singletonList(randomAlphaOfLength(4));
+            List<String> leaderIndexExclusionsPatterns = Collections.singletonList(randomAlphaOfLength(4));
             String followIndexNamePattern = randomAlphaOfLength(4);
             final Settings settings =
                 Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build();
@@ -89,6 +90,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
                 new AutoFollowMetadata.AutoFollowPattern(
                     remoteCluster,
                     leaderIndexPatterns,
+                    leaderIndexExclusionsPatterns,
                     followIndexNamePattern,
                     settings,
                     active,
@@ -124,6 +126,7 @@ protected void assertInstances(GetAutoFollowPatternAction.Response serverTestIns
             assertThat(serverPattern.getRemoteCluster(), equalTo(clientPattern.getRemoteCluster()));
             assertThat(serverPattern.getLeaderIndexPatterns(), equalTo(clientPattern.getLeaderIndexPatterns()));
             assertThat(serverPattern.getFollowIndexPattern(), equalTo(clientPattern.getFollowIndexNamePattern()));
+            assertThat(serverPattern.getLeaderIndexExclusionPatterns(), equalTo(clientPattern.getLeaderIndexExclusionPatterns()));
             assertThat(serverPattern.getSettings(), equalTo(clientPattern.getSettings()));
             assertThat(serverPattern.getMaxOutstandingReadRequests(), equalTo(clientPattern.getMaxOutstandingReadRequests()));
             assertThat(serverPattern.getMaxOutstandingWriteRequests(), equalTo(clientPattern.getMaxOutstandingWriteRequests()));

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java

@@ -28,7 +28,10 @@ public class PutAutoFollowPatternRequestTests extends AbstractRequestTestCase<
     protected PutAutoFollowPatternRequest createClientTestInstance() {
         // Name isn't serialized, because it specified in url path, so no need to randomly generate it here.
         PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("name",
-            randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
+            randomAlphaOfLength(4),
+            Arrays.asList(generateRandomStringArray(4, 4, false)),
+            Arrays.asList(generateRandomStringArray(4, 4, false))
+        );
         if (randomBoolean()) {
             putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
         }
@@ -75,6 +78,7 @@ protected void assertInstances(PutAutoFollowPatternAction.Request serverInstance
         assertThat(serverInstance.getName(), equalTo(clientTestInstance.getName()));
         assertThat(serverInstance.getRemoteCluster(), equalTo(clientTestInstance.getRemoteCluster()));
         assertThat(serverInstance.getLeaderIndexPatterns(), equalTo(clientTestInstance.getLeaderIndexPatterns()));
+        assertThat(serverInstance.getLeaderIndexExclusionPatterns(), equalTo(clientTestInstance.getLeaderIndexExclusionPatterns()));
         assertThat(serverInstance.getFollowIndexNamePattern(), equalTo(clientTestInstance.getFollowIndexNamePattern()));
         assertFollowConfig(serverInstance.getParameters(), clientTestInstance);
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java

@@ -474,12 +474,13 @@ public void testPutAutoFollowPattern() throws Exception {
             new PutAutoFollowPatternRequest(
                 "my_pattern", // <1>
                 "local", // <2>
-                Arrays.asList("logs-*", "metrics-*") // <3>
+                Arrays.asList("logs-*", "metrics-*"), // <3>
+                Arrays.asList("logs-excluded", "metrics-excluded") // <4>
         );
-        request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4>
+        request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <5>
         Settings settings =
             Settings.builder().put("index.number_of_replicas", 0L).build();
-        request.setSettings(settings); // <5>
+        request.setSettings(settings); // <6>
         // end::ccr-put-auto-follow-pattern-request
 
         // tag::ccr-put-auto-follow-pattern-execute

docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc

@@ -21,8 +21,9 @@ include-tagged::{doc-tests-file}[{api}-request]
 <1> The name of the auto follow pattern.
 <2> The name of the remote cluster.
 <3> The leader index patterns.
-<4> The pattern used to create the follower index
-<5> The settings overrides for the follower index
+<4> The leader index exclusion patterns.
+<5> The pattern used to create the follower index.
+<6> The settings overrides for the follower index.
 
 [id="{upid}-{api}-response"]
 ==== Response

docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc

@@ -96,6 +96,7 @@ The API returns the following result:
         [
           "leader_index*"
         ],
+        "leader_index_exclusion_patterns": [],
         "follow_index_pattern" : "{{leader_index}}-follower"
       }
     }

docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc

@@ -20,13 +20,18 @@ PUT /_ccr/auto_follow/<auto_follow_pattern_name>
   [
     "<leader_index_pattern>"
   ],
+  "leader_index_exclusion_patterns":
+  [
+    "leader_index_exclusion_pattern>"
+  ],
   "follow_index_pattern" : "<follow_index_pattern>"
 }
 --------------------------------------------------
 // TEST[setup:remote_cluster]
 // TEST[s/<auto_follow_pattern_name>/auto_follow_pattern_name/]
 // TEST[s/<remote_cluster>/remote_cluster/]
 // TEST[s/<leader_index_patterns>/leader_index*/]
+// TEST[s/<leader_index_exclusion_pattern>//]
 // TEST[s/<follow_index_pattern>/{{leader_index}}-follower/]
 
 //////////////////////////
@@ -72,6 +77,10 @@ indices.
   (Optional, array) An array of simple index patterns to match against indices
   in the remote cluster specified by the `remote_cluster` field.
 
+`leader_index_exclusion_patterns`::
+  (Optional, array) An array of simple index patterns to avoid following
+  indices in the remote cluster specified by the `remote_cluster` field.
+
 `follow_index_pattern`::
   (Optional, string) The name of follower index. The template `{{leader_index}}`
   can be used to derive the name of the follower index from the name of the

x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java

@@ -174,6 +174,64 @@ public void testAutoFollowPatterns() throws Exception {
         }
     }
 
+    public void testAutoFollowExclusionPatterns() throws Exception {
+        if ("follow".equals(targetCluster) == false) {
+            logger.info("skipping test, waiting for target cluster [follow]");
+            return;
+        }
+
+        final String autoFollowPatternName = getTestName().toLowerCase(Locale.ROOT);
+        try {
+            final String excludedIndex = "metrics-20220102";
+            int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices();
+            Request request = new Request("PUT", "/_ccr/auto_follow/" + autoFollowPatternName);
+            try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) {
+                bodyBuilder.startObject();
+                {
+                    bodyBuilder.startArray("leader_index_patterns");
+                    {
+                        bodyBuilder.value("metrics-*");
+                    }
+                    bodyBuilder.endArray();
+                    bodyBuilder.startArray("leader_index_exclusion_patterns");
+                    {
+                        bodyBuilder.value(excludedIndex);
+                    }
+                    bodyBuilder.endArray();
+                    bodyBuilder.field("remote_cluster", "leader_cluster");
+                }
+                bodyBuilder.endObject();
+                request.setJsonEntity(Strings.toString(bodyBuilder));
+            }
+            assertOK(client().performRequest(request));
+
+            try (RestClient leaderClient = buildLeaderClient()) {
+                request = new Request("PUT", "/metrics-20220101");
+                request.setJsonEntity("{\"mappings\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}");
+                assertOK(leaderClient.performRequest(request));
+            }
+
+            assertBusy(() -> {
+                assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1));
+                ensureYellow("metrics-20220101");
+            });
+
+            try (RestClient leaderClient = buildLeaderClient()) {
+                request = new Request("PUT", "/" + excludedIndex);
+                request.setJsonEntity("{\"mappings\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}");
+                assertOK(leaderClient.performRequest(request));
+            }
+
+            assertBusy(() -> {
+                assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1));
+            });
+
+            assertThat(indexExists(excludedIndex), is(false));
+        } finally {
+            cleanUpFollower(List.of("metrics-20220101"), List.of(), List.of(autoFollowPatternName));
+        }
+    }
+
     public void testPutAutoFollowPatternThatOverridesRequiredLeaderSetting() throws IOException {
         if ("follow".equals(targetCluster) == false) {
             logger.info("skipping test, waiting for target cluster [follow]");