Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for index pattern exclusion in CCR AutoFollow #72935

Merged
merged 11 commits into from
Jun 4, 2021
Merged
Original file line number Diff line number Diff line change
@@ -82,11 +82,19 @@ public static class Pattern extends FollowConfig {

@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<Pattern, Void> PARSER = new ConstructingObjectParser<>(
"pattern", true, args -> new Pattern((String) args[0], (List<String>) args[1], (String) args[2]));
"pattern",
true,
args -> new Pattern((String) args[0],
(List<String>) args[1],
args[2] == null ? Collections.emptyList() : (List<String>) args[2],
(String) args[3])
);

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD);
PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(),
PutAutoFollowPatternRequest.LEADER_EXCLUSION_PATTERNS_FIELD);
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD);
PARSER.declareObject(Pattern::setSettings, (p, c) -> Settings.fromXContent(p), PutAutoFollowPatternRequest.SETTINGS);
PARSER.declareInt(Pattern::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT);
@@ -123,11 +131,16 @@ public static class Pattern extends FollowConfig {

private final String remoteCluster;
private final List<String> leaderIndexPatterns;
private final List<String> leaderIndexExclusionPatterns;
private final String followIndexNamePattern;

Pattern(String remoteCluster, List<String> leaderIndexPatterns, String followIndexNamePattern) {
Pattern(String remoteCluster,
List<String> leaderIndexPatterns,
List<String> leaderIndexExclusionPatterns,
String followIndexNamePattern) {
this.remoteCluster = remoteCluster;
this.leaderIndexPatterns = leaderIndexPatterns;
this.leaderIndexExclusionPatterns = leaderIndexExclusionPatterns;
this.followIndexNamePattern = followIndexNamePattern;
}

@@ -139,6 +152,10 @@ public List<String> getLeaderIndexPatterns() {
return leaderIndexPatterns;
}

public List<String> getLeaderIndexExclusionPatterns() {
return leaderIndexExclusionPatterns;
}

public String getFollowIndexNamePattern() {
return followIndexNamePattern;
}
@@ -151,6 +168,7 @@ public boolean equals(Object o) {
Pattern pattern = (Pattern) o;
return Objects.equals(remoteCluster, pattern.remoteCluster) &&
Objects.equals(leaderIndexPatterns, pattern.leaderIndexPatterns) &&
Objects.equals(leaderIndexExclusionPatterns, pattern.leaderIndexExclusionPatterns) &&
Objects.equals(followIndexNamePattern, pattern.followIndexNamePattern);
}

@@ -160,6 +178,7 @@ public int hashCode() {
super.hashCode(),
remoteCluster,
leaderIndexPatterns,
leaderIndexExclusionPatterns,
followIndexNamePattern
);
}
Original file line number Diff line number Diff line change
@@ -14,23 +14,36 @@
import org.elasticsearch.common.xcontent.XContentBuilder;

import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;

public final class PutAutoFollowPatternRequest extends FollowConfig implements Validatable, ToXContentObject {

static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns");
static final ParseField LEADER_EXCLUSION_PATTERNS_FIELD = new ParseField("leader_index_exclusion_patterns");
static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern");

private final String name;
private final String remoteCluster;
private final List<String> leaderIndexPatterns;
private final List<String> leaderIndexExclusionPatterns;
private String followIndexNamePattern;

public PutAutoFollowPatternRequest(String name, String remoteCluster, List<String> leaderIndexPatterns) {
public PutAutoFollowPatternRequest(String name,
String remoteCluster,
List<String> leaderIndexPatterns) {
this(name, remoteCluster, leaderIndexPatterns, Collections.emptyList());
}

public PutAutoFollowPatternRequest(String name,
String remoteCluster,
List<String> leaderIndexPatterns,
List<String> leaderIndexExclusionPatterns) {
this.name = Objects.requireNonNull(name);
this.remoteCluster = Objects.requireNonNull(remoteCluster);
this.leaderIndexPatterns = Objects.requireNonNull(leaderIndexPatterns);
this.leaderIndexExclusionPatterns = Objects.requireNonNull(leaderIndexExclusionPatterns);
}

public String getName() {
@@ -45,6 +58,10 @@ public List<String> getLeaderIndexPatterns() {
return leaderIndexPatterns;
}

public List<String> getLeaderIndexExclusionPatterns() {
return leaderIndexExclusionPatterns;
}

public String getFollowIndexNamePattern() {
return followIndexNamePattern;
}
@@ -58,6 +75,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject();
builder.field(PutFollowRequest.REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster);
builder.field(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns);
if (leaderIndexExclusionPatterns.isEmpty() == false) {
builder.field(LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns);
}
if (followIndexNamePattern != null) {
builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern);
}
@@ -75,6 +95,7 @@ public boolean equals(Object o) {
return Objects.equals(name, that.name) &&
Objects.equals(remoteCluster, that.remoteCluster) &&
Objects.equals(leaderIndexPatterns, that.leaderIndexPatterns) &&
Objects.equals(leaderIndexExclusionPatterns, that.leaderIndexExclusionPatterns) &&
Objects.equals(followIndexNamePattern, that.followIndexNamePattern);
}

@@ -85,6 +106,7 @@ public int hashCode() {
name,
remoteCluster,
leaderIndexPatterns,
leaderIndexExclusionPatterns,
followIndexNamePattern
);
}
Original file line number Diff line number Diff line change
@@ -58,6 +58,7 @@
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;

public class CCRIT extends ESRestHighLevelClientTestCase {

@@ -244,8 +245,10 @@ public void testForgetFollower() throws IOException {

public void testAutoFollowing() throws Exception {
CcrClient ccrClient = highLevelClient().ccr();
PutAutoFollowPatternRequest putAutoFollowPatternRequest =
new PutAutoFollowPatternRequest("pattern1", "local_cluster", Collections.singletonList("logs-*"));
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("pattern1",
"local_cluster",
Collections.singletonList("logs-*"),
Collections.singletonList("logs-excluded"));
putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}");
final int followerNumberOfReplicas = randomIntBetween(0, 4);
final Settings autoFollowerPatternSettings =
@@ -255,6 +258,11 @@ public void testAutoFollowing() throws Exception {
execute(putAutoFollowPatternRequest, ccrClient::putAutoFollowPattern, ccrClient::putAutoFollowPatternAsync);
assertThat(putAutoFollowPatternResponse.isAcknowledged(), is(true));

CreateIndexRequest createExcludedIndexRequest = new CreateIndexRequest("logs-excluded");
CreateIndexResponse createExcludedIndexResponse =
highLevelClient().indices().create(createExcludedIndexRequest, RequestOptions.DEFAULT);
assertThat(createExcludedIndexResponse.isAcknowledged(), is(true));

CreateIndexRequest createIndexRequest = new CreateIndexRequest("logs-20200101");
CreateIndexResponse response = highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
assertThat(response.isAcknowledged(), is(true));
@@ -264,11 +272,13 @@ public void testAutoFollowing() throws Exception {
CcrStatsResponse ccrStatsResponse = execute(ccrStatsRequest, ccrClient::getCcrStats, ccrClient::getCcrStatsAsync);
assertThat(ccrStatsResponse.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(1L));
assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-20200101"), notNullValue());
assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-excluded"), nullValue());
});
assertThat(indexExists("copy-logs-20200101"), is(true));
assertThat(
getIndexSettingsAsMap("copy-logs-20200101"),
hasEntry("index.number_of_replicas", Integer.toString(followerNumberOfReplicas)));
assertThat(indexExists("copy-logs-excluded"), is(false));

GetAutoFollowPatternRequest getAutoFollowPatternRequest =
randomBoolean() ? new GetAutoFollowPatternRequest("pattern1") : new GetAutoFollowPatternRequest();
@@ -279,6 +289,7 @@ public void testAutoFollowing() throws Exception {
assertThat(pattern, notNullValue());
assertThat(pattern.getRemoteCluster(), equalTo(putAutoFollowPatternRequest.getRemoteCluster()));
assertThat(pattern.getLeaderIndexPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexPatterns()));
assertThat(pattern.getLeaderIndexExclusionPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexExclusionPatterns()));
assertThat(pattern.getFollowIndexNamePattern(), equalTo(putAutoFollowPatternRequest.getFollowIndexNamePattern()));
assertThat(pattern.getSettings(), equalTo(autoFollowerPatternSettings));

Original file line number Diff line number Diff line change
@@ -101,7 +101,10 @@ public void testForgetFollower() throws IOException {

public void testPutAutofollowPattern() throws Exception {
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest(randomAlphaOfLength(4),
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
randomAlphaOfLength(4),
Arrays.asList(generateRandomStringArray(4, 4, false)),
Arrays.asList(generateRandomStringArray(4, 4, false))
);
if (randomBoolean()) {
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
}
Original file line number Diff line number Diff line change
@@ -39,6 +39,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
for (int i = 0; i < numPatterns; i++) {
String remoteCluster = randomAlphaOfLength(4);
List<String> leaderIndexPatterns = Collections.singletonList(randomAlphaOfLength(4));
List<String> leaderIndexExclusionsPatterns = randomList(0, randomIntBetween(1, 10), () -> randomAlphaOfLength(4));
String followIndexNamePattern = randomAlphaOfLength(4);
final Settings settings =
Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build();
@@ -89,6 +90,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
new AutoFollowMetadata.AutoFollowPattern(
remoteCluster,
leaderIndexPatterns,
leaderIndexExclusionsPatterns,
followIndexNamePattern,
settings,
active,
@@ -124,6 +126,7 @@ protected void assertInstances(GetAutoFollowPatternAction.Response serverTestIns
assertThat(serverPattern.getRemoteCluster(), equalTo(clientPattern.getRemoteCluster()));
assertThat(serverPattern.getLeaderIndexPatterns(), equalTo(clientPattern.getLeaderIndexPatterns()));
assertThat(serverPattern.getFollowIndexPattern(), equalTo(clientPattern.getFollowIndexNamePattern()));
assertThat(serverPattern.getLeaderIndexExclusionPatterns(), equalTo(clientPattern.getLeaderIndexExclusionPatterns()));
assertThat(serverPattern.getSettings(), equalTo(clientPattern.getSettings()));
assertThat(serverPattern.getMaxOutstandingReadRequests(), equalTo(clientPattern.getMaxOutstandingReadRequests()));
assertThat(serverPattern.getMaxOutstandingWriteRequests(), equalTo(clientPattern.getMaxOutstandingWriteRequests()));
Original file line number Diff line number Diff line change
@@ -28,7 +28,10 @@ public class PutAutoFollowPatternRequestTests extends AbstractRequestTestCase<
protected PutAutoFollowPatternRequest createClientTestInstance() {
// Name isn't serialized, because it specified in url path, so no need to randomly generate it here.
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("name",
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
randomAlphaOfLength(4),
Arrays.asList(generateRandomStringArray(4, 4, false)),
Arrays.asList(generateRandomStringArray(4, 4, false))
);
if (randomBoolean()) {
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
}
@@ -75,6 +78,7 @@ protected void assertInstances(PutAutoFollowPatternAction.Request serverInstance
assertThat(serverInstance.getName(), equalTo(clientTestInstance.getName()));
assertThat(serverInstance.getRemoteCluster(), equalTo(clientTestInstance.getRemoteCluster()));
assertThat(serverInstance.getLeaderIndexPatterns(), equalTo(clientTestInstance.getLeaderIndexPatterns()));
assertThat(serverInstance.getLeaderIndexExclusionPatterns(), equalTo(clientTestInstance.getLeaderIndexExclusionPatterns()));
assertThat(serverInstance.getFollowIndexNamePattern(), equalTo(clientTestInstance.getFollowIndexNamePattern()));
assertFollowConfig(serverInstance.getParameters(), clientTestInstance);
}
Original file line number Diff line number Diff line change
@@ -474,12 +474,13 @@ public void testPutAutoFollowPattern() throws Exception {
new PutAutoFollowPatternRequest(
"my_pattern", // <1>
"local", // <2>
Arrays.asList("logs-*", "metrics-*") // <3>
Arrays.asList("logs-*", "metrics-*"), // <3>
Arrays.asList("logs-excluded", "metrics-excluded") // <4>
);
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4>
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <5>
Settings settings =
Settings.builder().put("index.number_of_replicas", 0L).build();
request.setSettings(settings); // <5>
request.setSettings(settings); // <6>
// end::ccr-put-auto-follow-pattern-request

// tag::ccr-put-auto-follow-pattern-execute
Original file line number Diff line number Diff line change
@@ -21,8 +21,9 @@ include-tagged::{doc-tests-file}[{api}-request]
<1> The name of the auto follow pattern.
<2> The name of the remote cluster.
<3> The leader index patterns.
<4> The pattern used to create the follower index
<5> The settings overrides for the follower index
<4> The leader index exclusion patterns.
<5> The pattern used to create the follower index.
<6> The settings overrides for the follower index.

[id="{upid}-{api}-response"]
==== Response
Original file line number Diff line number Diff line change
@@ -22,6 +22,10 @@ PUT /_ccr/auto_follow/my_auto_follow_pattern
[
"leader_index*"
],
"leader_index_exclusion_patterns":
[
"leader_index_001"
],
"follow_index_pattern" : "{{leader_index}}-follower"
}
--------------------------------------------------
@@ -96,6 +100,10 @@ The API returns the following result:
[
"leader_index*"
],
"leader_index_exclusion_patterns":
[
"leader_index_001"
],
"follow_index_pattern" : "{{leader_index}}-follower"
}
}
Original file line number Diff line number Diff line change
@@ -20,13 +20,18 @@ PUT /_ccr/auto_follow/<auto_follow_pattern_name>
[
"<leader_index_pattern>"
],
"leader_index_exclusion_patterns":
[
"<leader_index_exclusion_pattern>"
],
"follow_index_pattern" : "<follow_index_pattern>"
}
--------------------------------------------------
// TEST[setup:remote_cluster]
// TEST[s/<auto_follow_pattern_name>/auto_follow_pattern_name/]
// TEST[s/<remote_cluster>/remote_cluster/]
// TEST[s/<leader_index_patterns>/leader_index*/]
// TEST[s/<leader_index_exclusion_pattern>//]
// TEST[s/<follow_index_pattern>/{{leader_index}}-follower/]

//////////////////////////
@@ -54,7 +59,10 @@ This API creates a new named collection of
<<ccr-auto-follow,auto-follow patterns>> against the remote cluster
specified in the request body. Newly created indices on the remote cluster
matching any of the specified patterns will be automatically configured as follower
indices.
indices. Additionally, this API can be used to update existing
<<ccr-auto-follow,auto-follow patterns>>. Note that follower indices that were configured automatically
before updating an auto-follow pattern will remain unchanged even if they don't match against
the new patterns.

[[ccr-put-auto-follow-pattern-path-parms]]
==== {api-path-parms-title}
@@ -72,6 +80,11 @@ indices.
(Optional, array) An array of simple index patterns to match against indices
in the remote cluster specified by the `remote_cluster` field.

`leader_index_exclusion_patterns`::
(Optional, array) An array of simple index patterns that can be used to exclude indices
from being auto-followed. Indices in the remote cluster whose names are matching one or
more `leader_index_patterns` and one or more `leader_index_exclusion_patterns` won't be followed.

`follow_index_pattern`::
(Optional, string) The name of follower index. The template `{{leader_index}}`
can be used to derive the name of the follower index from the name of the
Loading