From c0fc17eb3959c87dac40b4096f43a0c9f62e0aa5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sat, 22 Dec 2018 18:10:02 +0100 Subject: [PATCH 01/35] [ILM] Add unfollow action This change adds the unfollow action for CCR follower indices. This is needed for the shrink action in case an index is a follower index. This will give the follower index the opportunity to fully catch up with the leader index, pause index following and unfollow the leader index. After this the shrink action can safely perform the ilm shrink. The unfollow action needs to be added to the hot phase and acts as barrier for going to the next phase (warm or delete phases), so that follower indices are being unfollowed properly before indices are expected to go in read-only mode. This allows the force merge action to execute its steps safely. The unfollow action has three steps: * `wait-for-indexing-complete` step: waits for the index in question to get the `index.lifecycle.indexing_complete` setting be set to `true` * `wait-for-follow-shard-tasks` step: waits for all the shard follow tasks for the index being handled to report that the leader shard global checkpoint is equal to the follower shard global checkpoint. * `unfollow-index` step: actually performs the unfollow. This consists out of multiple operations being executed on the index being handled: pause index following, close index, unfollow and open index. (a follower index can only be unfollowed when it is closed, because the underlying engine is changed) In the case of the last two steps, if the index in being handled is a regular index then the steps acts as a no-op. Relates to #34648 --- .../xpack/core/XPackClientPlugin.java | 4 +- .../TimeseriesLifecycleType.java | 2 +- .../core/indexlifecycle/UnfollowAction.java | 96 ++++++++ .../UnfollowFollowIndexStep.java | 83 +++++++ .../WaitForFollowShardTasksStep.java | 178 +++++++++++++++ .../WaitForIndexingComplete.java | 85 +++++++ .../LifecyclePolicyMetadataTests.java | 6 +- .../indexlifecycle/LifecyclePolicyTests.java | 10 +- .../TimeseriesLifecycleTypeTests.java | 3 + .../indexlifecycle/UnfollowActionTests.java | 60 +++++ .../WaitForFollowShardTasksStepInfoTests.java | 70 ++++++ .../WaitForFollowShardTasksStepTests.java | 208 ++++++++++++++++++ .../WaitForIndexingCompleteInfoTests.java | 62 ++++++ .../WaitForIndexingCompleteTests.java | 89 ++++++++ .../action/PutLifecycleRequestTests.java | 7 +- .../xpack/indexlifecycle/CCRLifecycleIT.java | 187 ++++++++++++++++ .../xpack/indexlifecycle/IndexLifecycle.java | 4 +- .../IndexLifecycleMetadataTests.java | 7 +- 18 files changed, 1150 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepInfoTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteInfoTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java create mode 100644 x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRLifecycleIT.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index f40cf8eba57c6..25185647d04c6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -55,6 +55,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction; import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType; +import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction; import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction; @@ -423,7 +424,8 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java index 17c9eaf17c083..92442291ecc88 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java @@ -34,7 +34,7 @@ public class TimeseriesLifecycleType implements LifecycleType { public static final String TYPE = "timeseries"; static final List VALID_PHASES = Arrays.asList("hot", "warm", "cold", "delete"); - static final List ORDERED_VALID_HOT_ACTIONS = Collections.singletonList(RolloverAction.NAME); + static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(UnfollowAction.NAME, RolloverAction.NAME); static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(AllocateAction.NAME); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java new file mode 100644 index 0000000000000..64dda09723412 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public final class UnfollowAction implements LifecycleAction { + + public static final String NAME = "unfollow"; + + public UnfollowAction() {} + + @Override + public List toSteps(Client client, String phase, StepKey nextStepKey) { + StepKey indexingComplete = new StepKey(phase, NAME, WaitForIndexingComplete.NAME); + StepKey waitForFollowShardTasks = new StepKey(phase, NAME, WaitForFollowShardTasksStep.NAME); + StepKey unfollowIndex = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); + + WaitForIndexingComplete step1 = new WaitForIndexingComplete(indexingComplete, waitForFollowShardTasks); + WaitForFollowShardTasksStep step2 = new WaitForFollowShardTasksStep(waitForFollowShardTasks, unfollowIndex, client); + UnfollowFollowIndexStep step3 = new UnfollowFollowIndexStep(unfollowIndex, nextStepKey, client); + return Arrays.asList(step1, step2, step3); + } + + @Override + public List toStepKeys(String phase) { + StepKey indexingCompleteStep = new StepKey(phase, NAME, WaitForIndexingComplete.NAME); + StepKey waitForFollowShardTasksStep = new StepKey(phase, NAME, WaitForFollowShardTasksStep.NAME); + StepKey unfollowIndexStep = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); + return Arrays.asList(indexingCompleteStep, waitForFollowShardTasksStep, unfollowIndexStep); + } + + @Override + public boolean isSafeAction() { + // There are no settings to change, so therefor this action should be safe: + return true; + } + + @Override + public String getWriteableName() { + return NAME; + } + + public UnfollowAction(StreamInput in) throws IOException {} + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, UnfollowAction::new); + + public static UnfollowAction parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return 1; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + return true; + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java new file mode 100644 index 0000000000000..73f957d7a8f9b --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; +import org.elasticsearch.xpack.core.ccr.action.UnfollowAction; + +import java.util.Map; + +final class UnfollowFollowIndexStep extends AsyncActionStep { + + static final String NAME = "unfollow-index"; + + UnfollowFollowIndexStep(StepKey key, StepKey nextStepKey, Client client) { + super(key, nextStepKey, client); + } + + @Override + public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener) { + String followerIndex = indexMetaData.getIndex().getName(); + Map customIndexMetadata = indexMetaData.getCustomData("ccr"); + if (customIndexMetadata == null) { + listener.onResponse(true); + return; + } + + pauseFollowerIndex(followerIndex, listener); + } + + void pauseFollowerIndex(final String followerIndex, final Listener listener) { + PauseFollowAction.Request request = new PauseFollowAction.Request(followerIndex); + getClient().execute(PauseFollowAction.INSTANCE, request, ActionListener.wrap( + r -> { + assert r.isAcknowledged(); + closeFollowerIndex(followerIndex, listener); + }, + listener::onFailure + )); + } + + void closeFollowerIndex(final String followerIndex, final Listener listener) { + CloseIndexRequest closeIndexRequest = new CloseIndexRequest(followerIndex); + getClient().admin().indices().close(closeIndexRequest, ActionListener.wrap( + acknowledgedResponse -> { + assert acknowledgedResponse.isAcknowledged(); + unfollow(followerIndex, listener); + }, + listener::onFailure) + ); + } + + void unfollow(final String followerIndex, final Listener listener) { + UnfollowAction.Request request = new UnfollowAction.Request(followerIndex); + getClient().execute(UnfollowAction.INSTANCE, request, ActionListener.wrap( + r -> { + assert r.isAcknowledged(); + openIndex(followerIndex, listener); + }, + listener::onFailure + )); + } + + void openIndex(final String index, final Listener listener) { + OpenIndexRequest request = new OpenIndexRequest(index); + getClient().admin().indices().open(request, ActionListener.wrap( + openIndexResponse -> { + assert openIndexResponse.isAcknowledged(); + listener.onResponse(true); + }, + listener::onFailure + )); + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java new file mode 100644 index 0000000000000..a2e5ac90fe74e --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus; +import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +final class WaitForFollowShardTasksStep extends AsyncWaitStep { + + private static final Logger LOGGER = LogManager.getLogger(WaitForFollowShardTasksStep.class); + static final String NAME = "wait-for-follow-shard-tasks"; + + WaitForFollowShardTasksStep(StepKey key, StepKey nextStepKey, Client client) { + super(key, nextStepKey, client); + } + + @Override + public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) { + Map customIndexMetadata = indexMetaData.getCustomData("ccr"); + if (customIndexMetadata == null) { + listener.onResponse(true, null); + return; + } + + FollowStatsAction.StatsRequest request = new FollowStatsAction.StatsRequest(); + request.setIndices(new String[]{indexMetaData.getIndex().getName()}); + getClient().execute(FollowStatsAction.INSTANCE, request, + ActionListener.wrap(r -> handleResponse(r, listener), listener::onFailure)); + } + + void handleResponse(FollowStatsAction.StatsResponses responses, Listener listener) { + List unSyncedShardFollowStatuses = responses.getStatsResponses() + .stream() + .map(FollowStatsAction.StatsResponse::status) + .filter(shardFollowStatus -> shardFollowStatus.leaderGlobalCheckpoint() != shardFollowStatus.followerGlobalCheckpoint()) + .collect(Collectors.toList()); + + // Follow stats api needs to return stats for follower index and all shard follow tasks should be synced: + boolean conditionMet = responses.getStatsResponses().size() > 0 && unSyncedShardFollowStatuses.isEmpty(); + if (conditionMet) { + listener.onResponse(true, null); + } else { + List shardFollowTaskInfos = unSyncedShardFollowStatuses + .stream() + .map(status -> new Info.ShardFollowTaskInfo(status.followerIndex(), status.getShardId(), + status.leaderGlobalCheckpoint(), status.followerGlobalCheckpoint())) + .collect(Collectors.toList()); + LOGGER.error("Condition not met [{}]", Strings.toString(new Info(shardFollowTaskInfos))); + listener.onResponse(false, new Info(shardFollowTaskInfos)); + } + } + + static final class Info implements ToXContentObject { + + static final ParseField SHARD_FOLLOW_TASKS = new ParseField("shard_follow_tasks"); + static final ParseField MESSAGE = new ParseField("message"); + + private final List shardFollowTaskInfos; + + Info(List shardFollowTaskInfos) { + this.shardFollowTaskInfos = shardFollowTaskInfos; + } + + List getShardFollowTaskInfos() { + return shardFollowTaskInfos; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(SHARD_FOLLOW_TASKS.getPreferredName(), shardFollowTaskInfos); + String message = "Waiting for [" + shardFollowTaskInfos.size() + "] shard follow tasks to be in sync"; + builder.field(MESSAGE.getPreferredName(), message); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Info info = (Info) o; + return Objects.equals(shardFollowTaskInfos, info.shardFollowTaskInfos); + } + + @Override + public int hashCode() { + return Objects.hash(shardFollowTaskInfos); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + static final class ShardFollowTaskInfo implements ToXContentObject { + + static final ParseField FOLLOWER_INDEX_FIELD = new ParseField("follower_index"); + static final ParseField SHARD_ID_FIELD = new ParseField("shard_id"); + static final ParseField LEADER_GLOBAL_CHECKPOINT_FIELD = new ParseField("leader_global_checkpoint"); + static final ParseField FOLLOWER_GLOBAL_CHECKPOINT_FIELD = new ParseField("follower_global_checkpoint"); + + private final String followerIndex; + private final int shardId; + private final long leaderGlobalCheckpoint; + private final long followerGlobalCheckpoint; + + ShardFollowTaskInfo(String followerIndex, int shardId, long leaderGlobalCheckpoint, long followerGlobalCheckpoint) { + this.followerIndex = followerIndex; + this.shardId = shardId; + this.leaderGlobalCheckpoint = leaderGlobalCheckpoint; + this.followerGlobalCheckpoint = followerGlobalCheckpoint; + } + + String getFollowerIndex() { + return followerIndex; + } + + + int getShardId() { + return shardId; + } + + long getLeaderGlobalCheckpoint() { + return leaderGlobalCheckpoint; + } + + long getFollowerGlobalCheckpoint() { + return followerGlobalCheckpoint; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex); + builder.field(SHARD_ID_FIELD.getPreferredName(), shardId); + builder.field(LEADER_GLOBAL_CHECKPOINT_FIELD.getPreferredName(), leaderGlobalCheckpoint); + builder.field(FOLLOWER_GLOBAL_CHECKPOINT_FIELD.getPreferredName(), followerGlobalCheckpoint); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ShardFollowTaskInfo that = (ShardFollowTaskInfo) o; + return shardId == that.shardId && + leaderGlobalCheckpoint == that.leaderGlobalCheckpoint && + followerGlobalCheckpoint == that.followerGlobalCheckpoint && + Objects.equals(followerIndex, that.followerIndex); + } + + @Override + public int hashCode() { + return Objects.hash(followerIndex, shardId, leaderGlobalCheckpoint, followerGlobalCheckpoint); + } + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java new file mode 100644 index 0000000000000..bf559bf418bfb --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.Index; + +import java.io.IOException; +import java.util.Objects; + +final class WaitForIndexingComplete extends ClusterStateWaitStep { + + static final String NAME = "wait-for-indexing-complete"; + + WaitForIndexingComplete(StepKey key, StepKey nextStepKey) { + super(key, nextStepKey); + } + + @Override + public Result isConditionMet(Index index, ClusterState clusterState) { + // No need to check whether an index is a following index, just check whether indexing_complete setting has been set. + // (Either by user or rollover action) + IndexMetaData followerIndex = clusterState.metaData().getIndexSafe(index); + boolean indexingComplete = LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.get(followerIndex.getSettings()); + if (indexingComplete) { + return new Result(true, null); + } else { + return new Result(false, new Info(followerIndex.getSettings())); + } + } + + static final class Info implements ToXContentObject { + + static final ParseField MESSAGE_FIELD = new ParseField("message"); + static final ParseField INDEX_SETTINGS_FIELD = new ParseField("index_settings"); + + private final String message; + private final Settings indexSettings; + + Info(Settings indexSettings) { + this.message = "the [" + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE + "] setting has not been set to true"; + this.indexSettings = indexSettings; + } + + String getMessage() { + return message; + } + + Settings getIndexSettings() { + return indexSettings; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(MESSAGE_FIELD.getPreferredName(), message); + builder.startObject(INDEX_SETTINGS_FIELD.getPreferredName()); + indexSettings.toXContent(builder, params); + builder.endObject(); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Info info = (Info) o; + return Objects.equals(indexSettings, info.indexSettings); + } + + @Override + public int hashCode() { + return Objects.hash(indexSettings); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadataTests.java index 5cb75e132ce92..99328e5e40e8d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadataTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadataTests.java @@ -43,7 +43,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new) )); } @@ -58,7 +59,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) )); return new NamedXContentRegistry(entries); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTests.java index 9ee7ee8d0acd0..46f001bc6de55 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTests.java @@ -52,7 +52,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new) )); } @@ -67,7 +68,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) )); return new NamedXContentRegistry(entries); } @@ -112,6 +114,8 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Null return RolloverActionTests.randomInstance(); case ShrinkAction.NAME: return ShrinkActionTests.randomInstance(); + case UnfollowAction.NAME: + return new UnfollowAction(); default: throw new IllegalArgumentException("invalid action [" + action + "]"); }}; @@ -158,6 +162,8 @@ public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String l return RolloverActionTests.randomInstance(); case ShrinkAction.NAME: return ShrinkActionTests.randomInstance(); + case UnfollowAction.NAME: + return new UnfollowAction(); default: throw new IllegalArgumentException("invalid action [" + action + "]"); }}; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java index 2f0c2f8d18b33..c4e98ab43e29a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java @@ -38,6 +38,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase { private static final RolloverAction TEST_ROLLOVER_ACTION = new RolloverAction(new ByteSizeValue(1), null, null); private static final ShrinkAction TEST_SHRINK_ACTION = new ShrinkAction(1); private static final ReadOnlyAction TEST_READ_ONLY_ACTION = new ReadOnlyAction(); + private static final UnfollowAction TEST_UNFOLLOW_ACTION = new UnfollowAction(); public void testValidatePhases() { boolean invalid = randomBoolean(); @@ -476,6 +477,8 @@ private LifecycleAction getTestAction(String actionName) { return TEST_ROLLOVER_ACTION; case ShrinkAction.NAME: return TEST_SHRINK_ACTION; + case UnfollowAction.NAME: + return TEST_UNFOLLOW_ACTION; default: throw new IllegalArgumentException("unsupported timeseries phase action [" + actionName + "]"); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java new file mode 100644 index 0000000000000..3765ebf3d8c34 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +public class UnfollowActionTests extends AbstractActionTestCase { + + @Override + protected UnfollowAction doParseInstance(XContentParser parser) throws IOException { + return UnfollowAction.parse(parser); + } + + @Override + protected UnfollowAction createTestInstance() { + return new UnfollowAction(); + } + + @Override + protected Reader instanceReader() { + return UnfollowAction::new; + } + + public void testToSteps() { + UnfollowAction action = createTestInstance(); + String phase = randomAlphaOfLengthBetween(1, 10); + StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10)); + List steps = action.toSteps(null, phase, nextStepKey); + assertThat(steps, notNullValue()); + assertThat(steps.size(), equalTo(3)); + + StepKey expectedFirstStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForIndexingComplete.NAME); + StepKey expectedSecondStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForFollowShardTasksStep.NAME); + StepKey expectedThirdStepKey = new StepKey(phase, UnfollowAction.NAME, UnfollowFollowIndexStep.NAME); + + WaitForIndexingComplete firstStep = (WaitForIndexingComplete) steps.get(0); + assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey)); + assertThat(firstStep.getNextStepKey(), equalTo(expectedSecondStepKey)); + + WaitForFollowShardTasksStep secondStep = (WaitForFollowShardTasksStep) steps.get(1); + assertThat(secondStep.getKey(), equalTo(expectedSecondStepKey)); + assertThat(secondStep.getNextStepKey(), equalTo(expectedThirdStepKey)); + + UnfollowFollowIndexStep thirdStep = (UnfollowFollowIndexStep) steps.get(2); + assertThat(thirdStep.getKey(), equalTo(expectedThirdStepKey)); + assertThat(thirdStep.getNextStepKey(), equalTo(nextStepKey)); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepInfoTests.java new file mode 100644 index 0000000000000..483df7632e2a4 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepInfoTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.xpack.core.indexlifecycle.WaitForFollowShardTasksStep.Info; +import org.elasticsearch.xpack.core.indexlifecycle.WaitForFollowShardTasksStep.Info.ShardFollowTaskInfo; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class WaitForFollowShardTasksStepInfoTests extends AbstractXContentTestCase { + + private static final ConstructingObjectParser SHARD_FOLLOW_TASK_INFO_PARSER = + new ConstructingObjectParser<>( + "shard_follow_task_info_parser", + args -> new ShardFollowTaskInfo((String) args[0], (Integer) args[1], (Long) args[2], (Long) args[3]) + ); + + static { + SHARD_FOLLOW_TASK_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), ShardFollowTaskInfo.FOLLOWER_INDEX_FIELD); + SHARD_FOLLOW_TASK_INFO_PARSER.declareInt(ConstructingObjectParser.constructorArg(), ShardFollowTaskInfo.SHARD_ID_FIELD); + SHARD_FOLLOW_TASK_INFO_PARSER.declareLong(ConstructingObjectParser.constructorArg(), + ShardFollowTaskInfo.LEADER_GLOBAL_CHECKPOINT_FIELD); + SHARD_FOLLOW_TASK_INFO_PARSER.declareLong(ConstructingObjectParser.constructorArg(), + ShardFollowTaskInfo.FOLLOWER_GLOBAL_CHECKPOINT_FIELD); + } + + private static final ConstructingObjectParser INFO_PARSER = new ConstructingObjectParser<>( + "info_parser", + args -> { + @SuppressWarnings("unchecked") + Info info = new Info((List) args[0]); + return info; + } + ); + + static { + INFO_PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), SHARD_FOLLOW_TASK_INFO_PARSER, + Info.SHARD_FOLLOW_TASKS); + INFO_PARSER.declareString((i, s) -> {}, Info.MESSAGE); + } + + @Override + protected Info createTestInstance() { + int numInfos = randomIntBetween(0, 32); + List shardFollowTaskInfos = new ArrayList<>(numInfos); + for (int i = 0; i < numInfos; i++) { + shardFollowTaskInfos.add(new ShardFollowTaskInfo(randomAlphaOfLength(3), randomIntBetween(0, 10), + randomNonNegativeLong(), randomNonNegativeLong())); + } + return new Info(shardFollowTaskInfos); + } + + @Override + protected Info doParseInstance(XContentParser parser) throws IOException { + return INFO_PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java new file mode 100644 index 0000000000000..18722c65df600 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus; +import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction; +import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.IsNull.notNullValue; + +public class WaitForFollowShardTasksStepTests extends AbstractStepTestCase { + + @Override + protected WaitForFollowShardTasksStep createRandomInstance() { + StepKey stepKey = randomStepKey(); + StepKey nextStepKey = randomStepKey(); + return new WaitForFollowShardTasksStep(stepKey, nextStepKey, Mockito.mock(Client.class)); + } + + @Override + protected WaitForFollowShardTasksStep mutateInstance(WaitForFollowShardTasksStep instance) { + StepKey key = instance.getKey(); + StepKey nextKey = instance.getNextStepKey(); + + if (randomBoolean()) { + key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } else { + nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } + + return new WaitForFollowShardTasksStep(key, nextKey, instance.getClient()); + } + + @Override + protected WaitForFollowShardTasksStep copyInstance(WaitForFollowShardTasksStep instance) { + return new WaitForFollowShardTasksStep(instance.getKey(), instance.getNextStepKey(), instance.getClient()); + } + + public void testConditionMet() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom("ccr", Collections.emptyMap()) + .numberOfShards(2) + .numberOfReplicas(0) + .build(); + Client client = Mockito.mock(Client.class); + List statsResponses = Arrays.asList( + new FollowStatsAction.StatsResponse(createShardFollowTaskStatus(0, 9, 9)), + new FollowStatsAction.StatsResponse(createShardFollowTaskStatus(1, 3, 3)) + ); + mockFollowStatsCall(client, indexMetadata.getIndex().getName(), statsResponses); + + WaitForFollowShardTasksStep step = new WaitForFollowShardTasksStep(randomStepKey(), randomStepKey(), client); + final boolean[] conditionMetHolder = new boolean[1]; + final ToXContentObject[] informationContextHolder = new ToXContentObject[1]; + final Exception[] exceptionHolder = new Exception[1]; + step.evaluateCondition(indexMetadata, new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject informationContext) { + conditionMetHolder[0] = conditionMet; + informationContextHolder[0] = informationContext; + } + + @Override + public void onFailure(Exception e) { + exceptionHolder[0] = e; + } + }); + + assertThat(conditionMetHolder[0], is(true)); + assertThat(informationContextHolder[0], nullValue()); + assertThat(exceptionHolder[0], nullValue()); + } + + public void testConditionNotMetShardsNotInSync() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom("ccr", Collections.emptyMap()) + .numberOfShards(2) + .numberOfReplicas(0) + .build(); + Client client = Mockito.mock(Client.class); + List statsResponses = Arrays.asList( + new FollowStatsAction.StatsResponse(createShardFollowTaskStatus(0, 9, 9)), + new FollowStatsAction.StatsResponse(createShardFollowTaskStatus(1, 8, 3)) + ); + mockFollowStatsCall(client, indexMetadata.getIndex().getName(), statsResponses); + + WaitForFollowShardTasksStep step = new WaitForFollowShardTasksStep(randomStepKey(), randomStepKey(), client); + final boolean[] conditionMetHolder = new boolean[1]; + final ToXContentObject[] informationContextHolder = new ToXContentObject[1]; + final Exception[] exceptionHolder = new Exception[1]; + step.evaluateCondition(indexMetadata, new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject informationContext) { + conditionMetHolder[0] = conditionMet; + informationContextHolder[0] = informationContext; + } + + @Override + public void onFailure(Exception e) { + exceptionHolder[0] = e; + } + }); + + assertThat(conditionMetHolder[0], is(false)); + assertThat(informationContextHolder[0], notNullValue()); + assertThat(exceptionHolder[0], nullValue()); + WaitForFollowShardTasksStep.Info info = (WaitForFollowShardTasksStep.Info) informationContextHolder[0]; + assertThat(info.getShardFollowTaskInfos().size(), equalTo(1)); + assertThat(info.getShardFollowTaskInfos().get(0).getShardId(), equalTo(1)); + assertThat(info.getShardFollowTaskInfos().get(0).getLeaderGlobalCheckpoint(), equalTo(8L)); + assertThat(info.getShardFollowTaskInfos().get(0).getFollowerGlobalCheckpoint(), equalTo(3L)); + } + + public void testConditionNotMetNotAFollowerIndex() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .numberOfShards(2) + .numberOfReplicas(0) + .build(); + Client client = Mockito.mock(Client.class); + + WaitForFollowShardTasksStep step = new WaitForFollowShardTasksStep(randomStepKey(), randomStepKey(), client); + final boolean[] conditionMetHolder = new boolean[1]; + final ToXContentObject[] informationContextHolder = new ToXContentObject[1]; + final Exception[] exceptionHolder = new Exception[1]; + step.evaluateCondition(indexMetadata, new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject informationContext) { + conditionMetHolder[0] = conditionMet; + informationContextHolder[0] = informationContext; + } + + @Override + public void onFailure(Exception e) { + exceptionHolder[0] = e; + } + }); + + assertThat(conditionMetHolder[0], is(true)); + assertThat(informationContextHolder[0], nullValue()); + assertThat(exceptionHolder[0], nullValue()); + } + + private static ShardFollowNodeTaskStatus createShardFollowTaskStatus(int shardId, long leaderGCP, long followerGCP) { + return new ShardFollowNodeTaskStatus( + "remote", + "leader-index", + "follower-index", + shardId, + leaderGCP, + -1, + followerGCP, + -1, + -1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + Collections.emptyNavigableMap(), + 0, + null + ); + } + + private void mockFollowStatsCall(Client client, String expectedIndexName, List statsResponses) { + Mockito.doAnswer(invocationOnMock -> { + FollowStatsAction.StatsRequest request = (FollowStatsAction.StatsRequest) invocationOnMock.getArguments()[1]; + assertThat(request.indices().length, equalTo(1)); + assertThat(request.indices()[0], equalTo(expectedIndexName)); + + @SuppressWarnings("unchecked") + ActionListener listener = + (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(new FollowStatsAction.StatsResponses(Collections.emptyList(), Collections.emptyList(), statsResponses)); + return null; + }).when(client).execute(Mockito.eq(FollowStatsAction.INSTANCE), Mockito.any(), Mockito.any()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteInfoTests.java new file mode 100644 index 0000000000000..c138148f11e4f --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteInfoTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.xpack.core.indexlifecycle.WaitForIndexingComplete.Info; + +import java.io.IOException; + +import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings; + +public class WaitForIndexingCompleteInfoTests extends AbstractXContentTestCase { + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("wait_for_indexing_complete_info", + args -> new Info((Settings) args[0])); + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), + (p, c) -> Settings.fromXContent(p), Info.INDEX_SETTINGS_FIELD); + PARSER.declareString((i, s) -> {}, Info.MESSAGE_FIELD); + } + + @Override + protected Info createTestInstance() { + return new Info(randomIndexSettings()); + } + + @Override + protected Info doParseInstance(XContentParser parser) throws IOException { + return PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + public final void testEqualsAndHashcode() { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copyInstance, this::mutateInstance); + } + } + + protected final Info copyInstance(Info instance) throws IOException { + return new Info(instance.getIndexSettings()); + } + + protected Info mutateInstance(Info instance) throws IOException { + Settings.Builder newSettings = Settings.builder().put(instance.getIndexSettings()); + newSettings.put(randomAlphaOfLength(4), randomAlphaOfLength(4)); + return new Info(newSettings.build()); + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java new file mode 100644 index 0000000000000..3db3a6b49a8ea --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class WaitForIndexingCompleteTests extends AbstractStepTestCase { + + @Override + protected WaitForIndexingComplete createRandomInstance() { + StepKey stepKey = randomStepKey(); + StepKey nextStepKey = randomStepKey(); + return new WaitForIndexingComplete(stepKey, nextStepKey); + } + + @Override + protected WaitForIndexingComplete mutateInstance(WaitForIndexingComplete instance) { + StepKey key = instance.getKey(); + StepKey nextKey = instance.getNextStepKey(); + + switch (between(0, 1)) { + case 0: + key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 1: + nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + + return new WaitForIndexingComplete(key, nextKey); + } + + @Override + protected WaitForIndexingComplete copyInstance(WaitForIndexingComplete instance) { + return new WaitForIndexingComplete(instance.getKey(), instance.getNextStepKey()); + } + + public void testConditionMet() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")) + .metaData(MetaData.builder().put(indexMetadata, true).build()) + .build(); + + WaitForIndexingComplete step = createRandomInstance(); + ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); + assertThat(result.isComplete(), is(true)); + assertThat(result.getInfomationContext(), nullValue()); + } + + public void testConditionNotMet() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "false")) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")) + .metaData(MetaData.builder().put(indexMetadata, true).build()) + .build(); + + WaitForIndexingComplete step = createRandomInstance(); + ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); + assertThat(result.isComplete(), is(false)); + assertThat(result.getInfomationContext(), notNullValue()); + WaitForIndexingComplete.Info info = (WaitForIndexingComplete.Info) result.getInfomationContext(); + assertThat(info.getIndexSettings(), equalTo(indexMetadata.getSettings())); + assertThat(info.getMessage(), equalTo("the [index.lifecycle.indexing_complete] setting has not been set to true")); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleRequestTests.java index 5df60a7333143..0003a75eafdfe 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleRequestTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction; import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType; +import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction; import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Request; import org.junit.Before; @@ -64,7 +65,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new) )); } @@ -79,7 +81,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) )); return new NamedXContentRegistry(entries); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRLifecycleIT.java new file mode 100644 index 0000000000000..d0b2dc0bb4216 --- /dev/null +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRLifecycleIT.java @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.indexlifecycle; + +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class CCRLifecycleIT extends ESRestTestCase { + + private static final Logger LOGGER = LogManager.getLogger(CCRLifecycleIT.class); + + public void testBasicCCRAndILMIntegration() throws Exception { + setupLocalRemoteCluster(); + putILMPolicy(); + + String leaderIndex = "logs-leader-1"; + String followerIndex = "logs-follower-1"; + + Settings indexSettings = Settings.builder() + .put("index.soft_deletes.enabled", true) + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.lifecycle.name", "my_policy") + .build(); + createIndex(leaderIndex, indexSettings); + ensureGreen(leaderIndex); + followIndex(leaderIndex, followerIndex); + + index(leaderIndex, "1"); + assertDocumentExists(leaderIndex, "1"); + assertBusy(() -> { + assertDocumentExists(followerIndex, "1"); + // Sanity check that following_index setting has been set, so that we can verify later that this setting has been unset: + assertThat(getIndexSetting(followerIndex, "index.xpack.ccr.following_index"), equalTo("true")); + + assertILMPolicy(leaderIndex, "hot"); + assertILMPolicy(followerIndex, "hot"); + }); + + updateIndexSettings(leaderIndex, Settings.builder() + .put("index.lifecycle.indexing_complete", true) + .build() + ); + assertBusy(() -> { + assertILMPolicy(leaderIndex, "warm"); + assertILMPolicy(followerIndex, "warm"); + + // ILM should have placed both indices in the warm phase and there these indices are read-only: + assertThat(getIndexSetting(leaderIndex, "index.blocks.write"), equalTo("true")); + assertThat(getIndexSetting(followerIndex, "index.blocks.write"), equalTo("true")); + // ILM should have unfollowed the follower index, so the following_index setting should have been removed: + // (this controls whether the follow engine is used) + assertThat(getIndexSetting(followerIndex, "index.xpack.ccr.following_index"), nullValue()); + }); + } + + private static void putILMPolicy() throws IOException { + final Request request = new Request("PUT", "_ilm/policy/my_policy"); + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + { + builder.startObject("policy"); + { + builder.startObject("phases"); + { + builder.startObject("hot"); + { + builder.startObject("actions"); + { + builder.startObject("unfollow"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + builder.startObject("warm"); + { + builder.startObject("actions"); + { + builder.startObject("readonly"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + builder.startObject("delete"); + { + builder.field("min_age", "7d"); + builder.startObject("actions"); + { + builder.startObject("delete"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + request.setJsonEntity(Strings.toString(builder)); + assertOK(client().performRequest(request)); + } + + private static void assertILMPolicy(String index, String expectedPhase) throws IOException { + final Request request = new Request("GET", "/" + index + "/_ilm/explain"); + Map response = toMap(client().performRequest(request)); + LOGGER.info("response={}", response); + Map explanation = (Map) ((Map) response.get("indices")).get(index); + assertThat(explanation.get("managed"), is(true)); + assertThat(explanation.get("policy"), equalTo("my_policy")); + assertThat(explanation.get("phase"), equalTo(expectedPhase)); + } + + private static void followIndex(String leaderIndex, String followIndex) throws IOException { + final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow"); + request.setJsonEntity("{\"remote_cluster\": \"local\", \"leader_index\": \"" + leaderIndex + + "\", \"read_poll_timeout\": \"10ms\"}"); + assertOK(client().performRequest(request)); + } + + private static void updateIndexSettings(String index, Settings settings) throws IOException { + final Request request = new Request("PUT", "/" + index + "/_settings"); + request.setJsonEntity(Strings.toString(settings)); + assertOK(client().performRequest(request)); + } + + private static Object getIndexSetting(String index, String setting) throws IOException { + Request request = new Request("GET", "/" + index + "/_settings"); + request.addParameter("flat_settings", "true"); + Map response = toMap(client().performRequest(request)); + Map settings = (Map) ((Map) response.get(index)).get("settings"); + return settings.get(setting); + } + + private static void index(String index, String id) throws IOException { + Request request = new Request("POST", "/" + index + "/_doc/" + id); + request.setJsonEntity("{}"); + assertOK(client().performRequest(request)); + } + + private static void assertDocumentExists(String index, String id) throws IOException { + Request request = new Request("HEAD", "/" + index + "/_doc/" + id); + Response response = client().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + } + + private static void setupLocalRemoteCluster() throws IOException { + Request request = new Request("GET", "/_nodes"); + Map nodesResponse = (Map) toMap(client().performRequest(request)).get("nodes"); + // Select node info of first node (we don't know the node id): + nodesResponse = (Map) nodesResponse.get(nodesResponse.keySet().iterator().next()); + String transportAddress = (String) nodesResponse.get("transport_address"); + + LOGGER.info("Configuring local remote cluster [{}]", transportAddress); + request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity("{\"persistent\": {\"cluster.remote.local.seeds\": \"" + transportAddress + "\"}}"); + assertThat(client().performRequest(request).getStatusLine().getStatusCode(), equalTo(200)); + } + + private static Map toMap(Response response) throws IOException { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); + } + +} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java index 27fb52fe2397f..3eba96db26f77 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java @@ -45,6 +45,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction; import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType; +import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction; import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction; @@ -157,7 +158,8 @@ public List getNa new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) ); } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java index 1c30ea841cb77..c03594bf25ddc 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction; import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType; +import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction; import java.io.IOException; import java.util.ArrayList; @@ -81,7 +82,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new), new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new), - new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new) + new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new), + new NamedWriteableRegistry.Entry(LifecycleAction.class, UnfollowAction.NAME, UnfollowAction::new) )); } @@ -96,7 +98,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) )); return new NamedXContentRegistry(entries); } From f9f530e276fb0e7f07b7cf7e64d93e7d7c9c22a5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 24 Dec 2018 11:09:02 +0100 Subject: [PATCH 02/35] Replaced ccr / ilm integ test with multi cluster integ test and changed WaitForIndexingComplete step to always ignore non follower indices. --- .../WaitForFollowShardTasksStep.java | 4 - .../WaitForIndexingComplete.java | 8 +- .../plugin/ilm/qa/multi-cluster/build.gradle | 54 +++++ .../indexlifecycle/CCRIndexLifecycleIT.java | 176 +++++++++++++++++ .../xpack/indexlifecycle/CCRLifecycleIT.java | 187 ------------------ 5 files changed, 236 insertions(+), 193 deletions(-) create mode 100644 x-pack/plugin/ilm/qa/multi-cluster/build.gradle create mode 100644 x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java delete mode 100644 x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRLifecycleIT.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java index a2e5ac90fe74e..31e09fdc9fd6c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java @@ -5,8 +5,6 @@ */ package org.elasticsearch.xpack.core.indexlifecycle; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -25,7 +23,6 @@ final class WaitForFollowShardTasksStep extends AsyncWaitStep { - private static final Logger LOGGER = LogManager.getLogger(WaitForFollowShardTasksStep.class); static final String NAME = "wait-for-follow-shard-tasks"; WaitForFollowShardTasksStep(StepKey key, StepKey nextStepKey, Client client) { @@ -63,7 +60,6 @@ void handleResponse(FollowStatsAction.StatsResponses responses, Listener listene .map(status -> new Info.ShardFollowTaskInfo(status.followerIndex(), status.getShardId(), status.leaderGlobalCheckpoint(), status.followerGlobalCheckpoint())) .collect(Collectors.toList()); - LOGGER.error("Condition not met [{}]", Strings.toString(new Info(shardFollowTaskInfos))); listener.onResponse(false, new Info(shardFollowTaskInfos)); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java index bf559bf418bfb..11970a9be703d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java @@ -14,6 +14,7 @@ import org.elasticsearch.index.Index; import java.io.IOException; +import java.util.Map; import java.util.Objects; final class WaitForIndexingComplete extends ClusterStateWaitStep { @@ -26,9 +27,12 @@ final class WaitForIndexingComplete extends ClusterStateWaitStep { @Override public Result isConditionMet(Index index, ClusterState clusterState) { - // No need to check whether an index is a following index, just check whether indexing_complete setting has been set. - // (Either by user or rollover action) IndexMetaData followerIndex = clusterState.metaData().getIndexSafe(index); + Map customIndexMetadata = followerIndex.getCustomData("ccr"); + if (customIndexMetadata == null) { + return new Result(true, null); + } + boolean indexingComplete = LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.get(followerIndex.getSettings()); if (indexingComplete) { return new Result(true, null); diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle new file mode 100644 index 0000000000000..59df733892944 --- /dev/null +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -0,0 +1,54 @@ +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.standalone-test' + +dependencies { + testCompile project(':x-pack:plugin:ccr:qa') +} + +task leaderClusterTest(type: RestIntegTestTask) { + mustRunAfter(precommit) +} + +leaderClusterTestCluster { + numNodes = 1 + clusterName = 'leader-cluster' + setting 'xpack.ilm.enabled', 'true' + setting 'xpack.ccr.enabled', 'true' + setting 'xpack.security.enabled', 'false' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' + setting 'indices.lifecycle.poll_interval', '1000ms' +} + +leaderClusterTestRunner { + systemProperty 'tests.target_cluster', 'leader' +} + +task followClusterTest(type: RestIntegTestTask) {} + +followClusterTestCluster { + dependsOn leaderClusterTestRunner + numNodes = 1 + clusterName = 'follow-cluster' + setting 'xpack.ilm.enabled', 'true' + setting 'xpack.ccr.enabled', 'true' + setting 'xpack.security.enabled', 'false' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' + setting 'indices.lifecycle.poll_interval', '1000ms' + setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\"" +} + +followClusterTestRunner { + systemProperty 'tests.target_cluster', 'follow' + systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}" + finalizedBy 'leaderClusterTestCluster#stop' +} + +check.dependsOn followClusterTest +unitTest.enabled = false // no unit tests for this module, only the rest integration test diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java new file mode 100644 index 0000000000000..7f9e9a96e4354 --- /dev/null +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.indexlifecycle; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ccr.ESCCRRestTestCase; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class CCRIndexLifecycleIT extends ESCCRRestTestCase { + + private static final Logger LOGGER = LogManager.getLogger(CCRIndexLifecycleIT.class); + + public void testBasicCCRAndILMIntegration() throws Exception { + String indexName = "logs-1"; + + if ("leader".equals(targetCluster)) { + putILMPolicy(); + Settings indexSettings = Settings.builder() + .put("index.soft_deletes.enabled", true) + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.lifecycle.name", "my_policy") + .put("index.lifecycle.rollover_alias", "logs") + .build(); + createIndex(indexName, indexSettings, "", "\"logs\": { }"); + ensureGreen(indexName); + } else if ("follow".equals(targetCluster)) { + // Policy with the same name must exist in follower cluster too: + putILMPolicy(); + followIndex(indexName, indexName); + // Aliases are not copied from leader index, so we need to add that for the rollover action in follower cluster: + client().performRequest(new Request("PUT", "/" + indexName + "/_alias/logs")); + + try (RestClient leaderClient = buildLeaderClient()) { + index(leaderClient, indexName, "1"); + assertDocumentExists(leaderClient, indexName, "1"); + + assertBusy(() -> { + assertDocumentExists(client(), indexName, "1"); + // Sanity check that following_index setting has been set, so that we can verify later that this setting has been unset: + assertThat(getIndexSetting(client(), indexName, "index.xpack.ccr.following_index"), equalTo("true")); + + assertILMPolicy(leaderClient, indexName, "hot"); + assertILMPolicy(client(), indexName, "hot"); + }); + + updateIndexSettings(leaderClient, indexName, Settings.builder() + .put("index.lifecycle.indexing_complete", true) + .build() + ); + + assertBusy(() -> { + // Ensure that 'index.lifecycle.indexing_complete' is replicated: + assertThat(getIndexSetting(leaderClient, indexName, "index.lifecycle.indexing_complete"), equalTo("true")); + assertThat(getIndexSetting(client(), indexName, "index.lifecycle.indexing_complete"), equalTo("true")); + + assertILMPolicy(leaderClient, indexName, "warm"); + assertILMPolicy(client(), indexName, "warm"); + + // ILM should have placed both indices in the warm phase and there these indices are read-only: + assertThat(getIndexSetting(leaderClient, indexName, "index.blocks.write"), equalTo("true")); + assertThat(getIndexSetting(client(), indexName, "index.blocks.write"), equalTo("true")); + // ILM should have unfollowed the follower index, so the following_index setting should have been removed: + // (this controls whether the follow engine is used) + assertThat(getIndexSetting(client(), indexName, "index.xpack.ccr.following_index"), nullValue()); + }); + } + } else { + fail("unexpected target cluster [" + targetCluster + "]"); + } + } + + private static void putILMPolicy() throws IOException { + final Request request = new Request("PUT", "_ilm/policy/my_policy"); + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + { + builder.startObject("policy"); + { + builder.startObject("phases"); + { + builder.startObject("hot"); + { + builder.startObject("actions"); + { + builder.startObject("rollover"); + builder.field("max_size", "50GB"); + builder.field("max_age", "7d"); + builder.endObject(); + } + { + builder.startObject("unfollow"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + builder.startObject("warm"); + { + builder.startObject("actions"); + { + builder.startObject("readonly"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + builder.startObject("delete"); + { + builder.field("min_age", "7d"); + builder.startObject("actions"); + { + builder.startObject("delete"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + request.setJsonEntity(Strings.toString(builder)); + assertOK(client().performRequest(request)); + } + + private static void assertILMPolicy(RestClient client, String index, String expectedPhase) throws IOException { + final Request request = new Request("GET", "/" + index + "/_ilm/explain"); + Map response = toMap(client.performRequest(request)); + LOGGER.info("response={}", response); + Map explanation = (Map) ((Map) response.get("indices")).get(index); + assertThat(explanation.get("managed"), is(true)); + assertThat(explanation.get("policy"), equalTo("my_policy")); + assertThat(explanation.get("phase"), equalTo(expectedPhase)); + } + + private static void updateIndexSettings(RestClient client, String index, Settings settings) throws IOException { + final Request request = new Request("PUT", "/" + index + "/_settings"); + request.setJsonEntity(Strings.toString(settings)); + assertOK(client.performRequest(request)); + } + + private static Object getIndexSetting(RestClient client, String index, String setting) throws IOException { + Request request = new Request("GET", "/" + index + "/_settings"); + request.addParameter("flat_settings", "true"); + Map response = toMap(client.performRequest(request)); + Map settings = (Map) ((Map) response.get(index)).get("settings"); + return settings.get(setting); + } + + private static void assertDocumentExists(RestClient client, String index, String id) throws IOException { + Request request = new Request("HEAD", "/" + index + "/_doc/" + id); + Response response = client.performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + } + +} diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRLifecycleIT.java deleted file mode 100644 index d0b2dc0bb4216..0000000000000 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRLifecycleIT.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.indexlifecycle; - -import org.apache.http.util.EntityUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.test.rest.ESRestTestCase; - -import java.io.IOException; -import java.util.Map; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; - -public class CCRLifecycleIT extends ESRestTestCase { - - private static final Logger LOGGER = LogManager.getLogger(CCRLifecycleIT.class); - - public void testBasicCCRAndILMIntegration() throws Exception { - setupLocalRemoteCluster(); - putILMPolicy(); - - String leaderIndex = "logs-leader-1"; - String followerIndex = "logs-follower-1"; - - Settings indexSettings = Settings.builder() - .put("index.soft_deletes.enabled", true) - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .put("index.lifecycle.name", "my_policy") - .build(); - createIndex(leaderIndex, indexSettings); - ensureGreen(leaderIndex); - followIndex(leaderIndex, followerIndex); - - index(leaderIndex, "1"); - assertDocumentExists(leaderIndex, "1"); - assertBusy(() -> { - assertDocumentExists(followerIndex, "1"); - // Sanity check that following_index setting has been set, so that we can verify later that this setting has been unset: - assertThat(getIndexSetting(followerIndex, "index.xpack.ccr.following_index"), equalTo("true")); - - assertILMPolicy(leaderIndex, "hot"); - assertILMPolicy(followerIndex, "hot"); - }); - - updateIndexSettings(leaderIndex, Settings.builder() - .put("index.lifecycle.indexing_complete", true) - .build() - ); - assertBusy(() -> { - assertILMPolicy(leaderIndex, "warm"); - assertILMPolicy(followerIndex, "warm"); - - // ILM should have placed both indices in the warm phase and there these indices are read-only: - assertThat(getIndexSetting(leaderIndex, "index.blocks.write"), equalTo("true")); - assertThat(getIndexSetting(followerIndex, "index.blocks.write"), equalTo("true")); - // ILM should have unfollowed the follower index, so the following_index setting should have been removed: - // (this controls whether the follow engine is used) - assertThat(getIndexSetting(followerIndex, "index.xpack.ccr.following_index"), nullValue()); - }); - } - - private static void putILMPolicy() throws IOException { - final Request request = new Request("PUT", "_ilm/policy/my_policy"); - XContentBuilder builder = jsonBuilder(); - builder.startObject(); - { - builder.startObject("policy"); - { - builder.startObject("phases"); - { - builder.startObject("hot"); - { - builder.startObject("actions"); - { - builder.startObject("unfollow"); - builder.endObject(); - } - builder.endObject(); - } - builder.endObject(); - builder.startObject("warm"); - { - builder.startObject("actions"); - { - builder.startObject("readonly"); - builder.endObject(); - } - builder.endObject(); - } - builder.endObject(); - builder.startObject("delete"); - { - builder.field("min_age", "7d"); - builder.startObject("actions"); - { - builder.startObject("delete"); - builder.endObject(); - } - builder.endObject(); - } - builder.endObject(); - } - builder.endObject(); - } - builder.endObject(); - } - builder.endObject(); - request.setJsonEntity(Strings.toString(builder)); - assertOK(client().performRequest(request)); - } - - private static void assertILMPolicy(String index, String expectedPhase) throws IOException { - final Request request = new Request("GET", "/" + index + "/_ilm/explain"); - Map response = toMap(client().performRequest(request)); - LOGGER.info("response={}", response); - Map explanation = (Map) ((Map) response.get("indices")).get(index); - assertThat(explanation.get("managed"), is(true)); - assertThat(explanation.get("policy"), equalTo("my_policy")); - assertThat(explanation.get("phase"), equalTo(expectedPhase)); - } - - private static void followIndex(String leaderIndex, String followIndex) throws IOException { - final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow"); - request.setJsonEntity("{\"remote_cluster\": \"local\", \"leader_index\": \"" + leaderIndex + - "\", \"read_poll_timeout\": \"10ms\"}"); - assertOK(client().performRequest(request)); - } - - private static void updateIndexSettings(String index, Settings settings) throws IOException { - final Request request = new Request("PUT", "/" + index + "/_settings"); - request.setJsonEntity(Strings.toString(settings)); - assertOK(client().performRequest(request)); - } - - private static Object getIndexSetting(String index, String setting) throws IOException { - Request request = new Request("GET", "/" + index + "/_settings"); - request.addParameter("flat_settings", "true"); - Map response = toMap(client().performRequest(request)); - Map settings = (Map) ((Map) response.get(index)).get("settings"); - return settings.get(setting); - } - - private static void index(String index, String id) throws IOException { - Request request = new Request("POST", "/" + index + "/_doc/" + id); - request.setJsonEntity("{}"); - assertOK(client().performRequest(request)); - } - - private static void assertDocumentExists(String index, String id) throws IOException { - Request request = new Request("HEAD", "/" + index + "/_doc/" + id); - Response response = client().performRequest(request); - assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); - } - - private static void setupLocalRemoteCluster() throws IOException { - Request request = new Request("GET", "/_nodes"); - Map nodesResponse = (Map) toMap(client().performRequest(request)).get("nodes"); - // Select node info of first node (we don't know the node id): - nodesResponse = (Map) nodesResponse.get(nodesResponse.keySet().iterator().next()); - String transportAddress = (String) nodesResponse.get("transport_address"); - - LOGGER.info("Configuring local remote cluster [{}]", transportAddress); - request = new Request("PUT", "/_cluster/settings"); - request.setJsonEntity("{\"persistent\": {\"cluster.remote.local.seeds\": \"" + transportAddress + "\"}}"); - assertThat(client().performRequest(request).getStatusLine().getStatusCode(), equalTo(200)); - } - - private static Map toMap(Response response) throws IOException { - return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); - } - -} From 80bb2efcc38a74e7166b1a7a367ea6d80a9fdbc3 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 24 Dec 2018 12:46:28 +0100 Subject: [PATCH 03/35] Fixed tests --- .../WaitForIndexingCompleteTests.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java index 3db3a6b49a8ea..d0085911fc120 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java @@ -10,8 +10,11 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; +import java.util.Collections; + import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -53,6 +56,24 @@ protected WaitForIndexingComplete copyInstance(WaitForIndexingComplete instance) public void testConditionMet() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom("ccr", Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")) + .metaData(MetaData.builder().put(indexMetadata, true).build()) + .build(); + + WaitForIndexingComplete step = createRandomInstance(); + ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); + assertThat(result.isComplete(), is(true)); + assertThat(result.getInfomationContext(), nullValue()); + } + + public void testConditionMetNotAFollowerIndex() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT)) .numberOfShards(1) .numberOfReplicas(0) .build(); @@ -68,8 +89,13 @@ public void testConditionMet() { } public void testConditionNotMet() { + Settings.Builder indexSettings = settings(Version.CURRENT); + if (randomBoolean()) { + indexSettings.put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "false"); + } IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") - .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "false")) + .settings(indexSettings) + .putCustom("ccr", Collections.emptyMap()) .numberOfShards(1) .numberOfReplicas(0) .build(); From a265c2a82c636e92978b9dce56b6d3bd30eb1655 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 24 Dec 2018 16:03:46 +0100 Subject: [PATCH 04/35] Added more tests --- .../UnfollowFollowIndexStepTests.java | 381 ++++++++++++++++++ .../WaitForFollowShardTasksStepTests.java | 1 + .../WaitForIndexingCompleteTests.java | 13 +- 3 files changed, 386 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java new file mode 100644 index 0000000000000..46f1d11a5eca5 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java @@ -0,0 +1,381 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.AdminClient; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; +import org.elasticsearch.xpack.core.ccr.action.UnfollowAction; +import org.mockito.Mockito; + +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; + +public class UnfollowFollowIndexStepTests extends AbstractStepTestCase { + + @Override + protected UnfollowFollowIndexStep createRandomInstance() { + Step.StepKey stepKey = randomStepKey(); + Step.StepKey nextStepKey = randomStepKey(); + return new UnfollowFollowIndexStep(stepKey, nextStepKey, Mockito.mock(Client.class)); + } + + @Override + protected UnfollowFollowIndexStep mutateInstance(UnfollowFollowIndexStep instance) { + Step.StepKey key = instance.getKey(); + Step.StepKey nextKey = instance.getNextStepKey(); + + if (randomBoolean()) { + key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } else { + nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } + + return new UnfollowFollowIndexStep(key, nextKey, instance.getClient()); + } + + @Override + protected UnfollowFollowIndexStep copyInstance(UnfollowFollowIndexStep instance) { + return new UnfollowFollowIndexStep(instance.getKey(), instance.getNextStepKey(), instance.getClient()); + } + + public void testUnFollow() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom("ccr", Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + // Mock pause follow api call: + Mockito.doAnswer(invocation -> { + PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + // Mock close index api call: + Mockito.doAnswer(invocation -> { + CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(indicesClient).close(Mockito.any(), Mockito.any()); + + // Mock unfollow api call: + Mockito.doAnswer(invocation -> { + UnfollowAction.Request request = (UnfollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowerIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + // Mock open index api call: + Mockito.doAnswer(invocation -> { + OpenIndexRequest closeIndexRequest = (OpenIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new OpenIndexResponse(true, true)); + return null; + }).when(indicesClient).open(Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], is(true)); + assertThat(failure[0], nullValue()); + } + + public void testUnFollowOpenIndexFailed() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom("ccr", Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + // Mock pause follow api call: + Mockito.doAnswer(invocation -> { + PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + // Mock close index api call: + Mockito.doAnswer(invocation -> { + CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(indicesClient).close(Mockito.any(), Mockito.any()); + + // Mock unfollow api call: + Mockito.doAnswer(invocation -> { + UnfollowAction.Request request = (UnfollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowerIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + // Fail open index api call: + Exception error = new RuntimeException(); + Mockito.doAnswer(invocation -> { + OpenIndexRequest closeIndexRequest = (OpenIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onFailure(error); + return null; + }).when(indicesClient).open(Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], nullValue()); + assertThat(failure[0], sameInstance(error)); + } + + public void testUnFollowUnfollowFailed() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom("ccr", Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + // Mock pause follow api call: + Mockito.doAnswer(invocation -> { + PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + // Mock close index api call: + Mockito.doAnswer(invocation -> { + CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(indicesClient).close(Mockito.any(), Mockito.any()); + + // Mock unfollow api call: + Exception error = new RuntimeException(); + Mockito.doAnswer(invocation -> { + UnfollowAction.Request request = (UnfollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowerIndex(), equalTo("follower-index")); + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onFailure(error); + return null; + }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], nullValue()); + assertThat(failure[0], sameInstance(error)); + } + + public void testUnFollowCloseIndexFailed() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom("ccr", Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + // Mock pause follow api call: + Mockito.doAnswer(invocation -> { + PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + // Mock close index api call: + Exception error = new RuntimeException(); + Mockito.doAnswer(invocation -> { + CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onFailure(error); + return null; + }).when(indicesClient).close(Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], nullValue()); + assertThat(failure[0], sameInstance(error)); + } + + public void testUnFollowPauseIndexFollowingFailed() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom("ccr", Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + // Mock pause follow api call: + Client client = Mockito.mock(Client.class); + Exception error = new RuntimeException(); + Mockito.doAnswer(invocation -> { + PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowIndex(), equalTo("follower-index")); + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onFailure(error); + return null; + }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], nullValue()); + assertThat(failure[0], sameInstance(error)); + Mockito.verify(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(client); + } + + public void testUnFollowNotAFollowerIndex() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], is(true)); + assertThat(failure[0], nullValue()); + Mockito.verifyZeroInteractions(client); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java index 18722c65df600..72725fb38a9e2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java @@ -157,6 +157,7 @@ public void onFailure(Exception e) { assertThat(conditionMetHolder[0], is(true)); assertThat(informationContextHolder[0], nullValue()); assertThat(exceptionHolder[0], nullValue()); + Mockito.verifyZeroInteractions(client); } private static ShardFollowNodeTaskStatus createShardFollowTaskStatus(int shardId, long leaderGCP, long followerGCP) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java index d0085911fc120..a342d5536d4a5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java @@ -34,15 +34,10 @@ protected WaitForIndexingComplete mutateInstance(WaitForIndexingComplete instanc StepKey key = instance.getKey(); StepKey nextKey = instance.getNextStepKey(); - switch (between(0, 1)) { - case 0: - key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); - break; - case 1: - nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); - break; - default: - throw new AssertionError("Illegal randomisation branch"); + if (randomBoolean()) { + key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } else { + nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); } return new WaitForIndexingComplete(key, nextKey); From e281b038a17c2aa7050102ceadcad0dc9e22f335 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 3 Jan 2019 11:10:08 -0700 Subject: [PATCH 05/35] Rename class to WaitForIndexingCompleteStep This aligns WaitForIndexingCompleteStep with naming conventions --- .../core/indexlifecycle/UnfollowAction.java | 6 ++--- ....java => WaitForIndexingCompleteStep.java} | 4 ++-- .../indexlifecycle/UnfollowActionTests.java | 4 ++-- ...WaitForIndexingCompleteStepInfoTests.java} | 4 ++-- ... => WaitForIndexingCompleteStepTests.java} | 22 +++++++++---------- 5 files changed, 20 insertions(+), 20 deletions(-) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/{WaitForIndexingComplete.java => WaitForIndexingCompleteStep.java} (95%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/{WaitForIndexingCompleteInfoTests.java => WaitForIndexingCompleteStepInfoTests.java} (95%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/{WaitForIndexingCompleteTests.java => WaitForIndexingCompleteStepTests.java} (81%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java index 64dda09723412..32a5e5d27e813 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java @@ -26,11 +26,11 @@ public UnfollowAction() {} @Override public List toSteps(Client client, String phase, StepKey nextStepKey) { - StepKey indexingComplete = new StepKey(phase, NAME, WaitForIndexingComplete.NAME); + StepKey indexingComplete = new StepKey(phase, NAME, WaitForIndexingCompleteStep.NAME); StepKey waitForFollowShardTasks = new StepKey(phase, NAME, WaitForFollowShardTasksStep.NAME); StepKey unfollowIndex = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); - WaitForIndexingComplete step1 = new WaitForIndexingComplete(indexingComplete, waitForFollowShardTasks); + WaitForIndexingCompleteStep step1 = new WaitForIndexingCompleteStep(indexingComplete, waitForFollowShardTasks); WaitForFollowShardTasksStep step2 = new WaitForFollowShardTasksStep(waitForFollowShardTasks, unfollowIndex, client); UnfollowFollowIndexStep step3 = new UnfollowFollowIndexStep(unfollowIndex, nextStepKey, client); return Arrays.asList(step1, step2, step3); @@ -38,7 +38,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { @Override public List toStepKeys(String phase) { - StepKey indexingCompleteStep = new StepKey(phase, NAME, WaitForIndexingComplete.NAME); + StepKey indexingCompleteStep = new StepKey(phase, NAME, WaitForIndexingCompleteStep.NAME); StepKey waitForFollowShardTasksStep = new StepKey(phase, NAME, WaitForFollowShardTasksStep.NAME); StepKey unfollowIndexStep = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); return Arrays.asList(indexingCompleteStep, waitForFollowShardTasksStep, unfollowIndexStep); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java similarity index 95% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java index 11970a9be703d..4e792f3f801d6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingComplete.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java @@ -17,11 +17,11 @@ import java.util.Map; import java.util.Objects; -final class WaitForIndexingComplete extends ClusterStateWaitStep { +final class WaitForIndexingCompleteStep extends ClusterStateWaitStep { static final String NAME = "wait-for-indexing-complete"; - WaitForIndexingComplete(StepKey key, StepKey nextStepKey) { + WaitForIndexingCompleteStep(StepKey key, StepKey nextStepKey) { super(key, nextStepKey); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java index 3765ebf3d8c34..afe6488da2903 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java @@ -41,11 +41,11 @@ public void testToSteps() { assertThat(steps, notNullValue()); assertThat(steps.size(), equalTo(3)); - StepKey expectedFirstStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForIndexingComplete.NAME); + StepKey expectedFirstStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForIndexingCompleteStep.NAME); StepKey expectedSecondStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForFollowShardTasksStep.NAME); StepKey expectedThirdStepKey = new StepKey(phase, UnfollowAction.NAME, UnfollowFollowIndexStep.NAME); - WaitForIndexingComplete firstStep = (WaitForIndexingComplete) steps.get(0); + WaitForIndexingCompleteStep firstStep = (WaitForIndexingCompleteStep) steps.get(0); assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey)); assertThat(firstStep.getNextStepKey(), equalTo(expectedSecondStepKey)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepInfoTests.java similarity index 95% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteInfoTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepInfoTests.java index c138148f11e4f..dcd29fb3b10e2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteInfoTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepInfoTests.java @@ -11,13 +11,13 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; -import org.elasticsearch.xpack.core.indexlifecycle.WaitForIndexingComplete.Info; +import org.elasticsearch.xpack.core.indexlifecycle.WaitForIndexingCompleteStep.Info; import java.io.IOException; import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings; -public class WaitForIndexingCompleteInfoTests extends AbstractXContentTestCase { +public class WaitForIndexingCompleteStepInfoTests extends AbstractXContentTestCase { private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("wait_for_indexing_complete_info", diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java similarity index 81% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java index a342d5536d4a5..c9e0701212002 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java @@ -20,17 +20,17 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class WaitForIndexingCompleteTests extends AbstractStepTestCase { +public class WaitForIndexingCompleteStepTests extends AbstractStepTestCase { @Override - protected WaitForIndexingComplete createRandomInstance() { + protected WaitForIndexingCompleteStep createRandomInstance() { StepKey stepKey = randomStepKey(); StepKey nextStepKey = randomStepKey(); - return new WaitForIndexingComplete(stepKey, nextStepKey); + return new WaitForIndexingCompleteStep(stepKey, nextStepKey); } @Override - protected WaitForIndexingComplete mutateInstance(WaitForIndexingComplete instance) { + protected WaitForIndexingCompleteStep mutateInstance(WaitForIndexingCompleteStep instance) { StepKey key = instance.getKey(); StepKey nextKey = instance.getNextStepKey(); @@ -40,12 +40,12 @@ protected WaitForIndexingComplete mutateInstance(WaitForIndexingComplete instanc nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); } - return new WaitForIndexingComplete(key, nextKey); + return new WaitForIndexingCompleteStep(key, nextKey); } @Override - protected WaitForIndexingComplete copyInstance(WaitForIndexingComplete instance) { - return new WaitForIndexingComplete(instance.getKey(), instance.getNextStepKey()); + protected WaitForIndexingCompleteStep copyInstance(WaitForIndexingCompleteStep instance) { + return new WaitForIndexingCompleteStep(instance.getKey(), instance.getNextStepKey()); } public void testConditionMet() { @@ -60,7 +60,7 @@ public void testConditionMet() { .metaData(MetaData.builder().put(indexMetadata, true).build()) .build(); - WaitForIndexingComplete step = createRandomInstance(); + WaitForIndexingCompleteStep step = createRandomInstance(); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); assertThat(result.isComplete(), is(true)); assertThat(result.getInfomationContext(), nullValue()); @@ -77,7 +77,7 @@ public void testConditionMetNotAFollowerIndex() { .metaData(MetaData.builder().put(indexMetadata, true).build()) .build(); - WaitForIndexingComplete step = createRandomInstance(); + WaitForIndexingCompleteStep step = createRandomInstance(); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); assertThat(result.isComplete(), is(true)); assertThat(result.getInfomationContext(), nullValue()); @@ -99,11 +99,11 @@ public void testConditionNotMet() { .metaData(MetaData.builder().put(indexMetadata, true).build()) .build(); - WaitForIndexingComplete step = createRandomInstance(); + WaitForIndexingCompleteStep step = createRandomInstance(); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); assertThat(result.isComplete(), is(false)); assertThat(result.getInfomationContext(), notNullValue()); - WaitForIndexingComplete.Info info = (WaitForIndexingComplete.Info) result.getInfomationContext(); + WaitForIndexingCompleteStep.Info info = (WaitForIndexingCompleteStep.Info) result.getInfomationContext(); assertThat(info.getIndexSettings(), equalTo(indexMetadata.getSettings())); assertThat(info.getMessage(), equalTo("the [index.lifecycle.indexing_complete] setting has not been set to true")); } From 07f0136d7935606c7f5829a8e03f8ed215cd0871 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 3 Jan 2019 11:23:22 -0700 Subject: [PATCH 06/35] Factor "ccr" out into constant --- .../xpack/core/indexlifecycle/UnfollowAction.java | 1 + .../core/indexlifecycle/UnfollowFollowIndexStep.java | 4 +++- .../indexlifecycle/WaitForFollowShardTasksStep.java | 4 +++- .../indexlifecycle/WaitForIndexingCompleteStep.java | 4 +++- .../indexlifecycle/UnfollowFollowIndexStepTests.java | 11 ++++++----- .../WaitForFollowShardTasksStepTests.java | 5 +++-- .../WaitForIndexingCompleteStepTests.java | 5 +++-- 7 files changed, 22 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java index 32a5e5d27e813..0d333b001a7bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java @@ -21,6 +21,7 @@ public final class UnfollowAction implements LifecycleAction { public static final String NAME = "unfollow"; + public static final String CCR_METADATA_KEY = "ccr"; public UnfollowAction() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java index 73f957d7a8f9b..42ee2353f9496 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java @@ -16,6 +16,8 @@ import java.util.Map; +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; + final class UnfollowFollowIndexStep extends AsyncActionStep { static final String NAME = "unfollow-index"; @@ -27,7 +29,7 @@ final class UnfollowFollowIndexStep extends AsyncActionStep { @Override public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener) { String followerIndex = indexMetaData.getIndex().getName(); - Map customIndexMetadata = indexMetaData.getCustomData("ccr"); + Map customIndexMetadata = indexMetaData.getCustomData(CCR_METADATA_KEY); if (customIndexMetadata == null) { listener.onResponse(true); return; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java index 31e09fdc9fd6c..2df8ad07538ef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java @@ -21,6 +21,8 @@ import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; + final class WaitForFollowShardTasksStep extends AsyncWaitStep { static final String NAME = "wait-for-follow-shard-tasks"; @@ -31,7 +33,7 @@ final class WaitForFollowShardTasksStep extends AsyncWaitStep { @Override public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) { - Map customIndexMetadata = indexMetaData.getCustomData("ccr"); + Map customIndexMetadata = indexMetaData.getCustomData(CCR_METADATA_KEY); if (customIndexMetadata == null) { listener.onResponse(true, null); return; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java index 4e792f3f801d6..4844239491bed 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java @@ -17,6 +17,8 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; + final class WaitForIndexingCompleteStep extends ClusterStateWaitStep { static final String NAME = "wait-for-indexing-complete"; @@ -28,7 +30,7 @@ final class WaitForIndexingCompleteStep extends ClusterStateWaitStep { @Override public Result isConditionMet(Index index, ClusterState clusterState) { IndexMetaData followerIndex = clusterState.metaData().getIndexSafe(index); - Map customIndexMetadata = followerIndex.getCustomData("ccr"); + Map customIndexMetadata = followerIndex.getCustomData(CCR_METADATA_KEY); if (customIndexMetadata == null) { return new Result(true, null); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java index 46f1d11a5eca5..c2323ed0f3bff 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java @@ -21,6 +21,7 @@ import java.util.Collections; +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -57,7 +58,7 @@ protected UnfollowFollowIndexStep copyInstance(UnfollowFollowIndexStep instance) public void testUnFollow() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(1) .numberOfReplicas(0) .build(); @@ -129,7 +130,7 @@ public void onFailure(Exception e) { public void testUnFollowOpenIndexFailed() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(1) .numberOfReplicas(0) .build(); @@ -201,7 +202,7 @@ public void onFailure(Exception e) { public void testUnFollowUnfollowFailed() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(1) .numberOfReplicas(0) .build(); @@ -263,7 +264,7 @@ public void onFailure(Exception e) { public void testUnFollowCloseIndexFailed() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(1) .numberOfReplicas(0) .build(); @@ -315,7 +316,7 @@ public void onFailure(Exception e) { public void testUnFollowPauseIndexFollowingFailed() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(1) .numberOfReplicas(0) .build(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java index 72725fb38a9e2..a0ee01a240347 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java @@ -19,6 +19,7 @@ import java.util.Collections; import java.util.List; +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -55,7 +56,7 @@ protected WaitForFollowShardTasksStep copyInstance(WaitForFollowShardTasksStep i public void testConditionMet() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(2) .numberOfReplicas(0) .build(); @@ -91,7 +92,7 @@ public void onFailure(Exception e) { public void testConditionNotMetShardsNotInSync() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(2) .numberOfReplicas(0) .build(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java index c9e0701212002..7654a6660bcf1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java @@ -15,6 +15,7 @@ import java.util.Collections; +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -51,7 +52,7 @@ protected WaitForIndexingCompleteStep copyInstance(WaitForIndexingCompleteStep i public void testConditionMet() { IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(1) .numberOfReplicas(0) .build(); @@ -90,7 +91,7 @@ public void testConditionNotMet() { } IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") .settings(indexSettings) - .putCustom("ccr", Collections.emptyMap()) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) .numberOfShards(1) .numberOfReplicas(0) .build(); From 509718c0c6c498fc2b7716b6cf80facb65f8e141 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 3 Jan 2019 11:29:58 -0700 Subject: [PATCH 07/35] Add note about leader index to error message --- .../xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java index 4844239491bed..bc52842d3ead9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java @@ -52,7 +52,8 @@ static final class Info implements ToXContentObject { private final Settings indexSettings; Info(Settings indexSettings) { - this.message = "the [" + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE + "] setting has not been set to true"; + this.message = "the [" + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE + + "] setting has not been set to true on the leader index"; this.indexSettings = indexSettings; } From 974bb8b5eab04a40b6a145e47c57fef6499f638a Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 3 Jan 2019 15:20:12 -0700 Subject: [PATCH 08/35] Factor out mocking into methods --- .../UnfollowFollowIndexStepTests.java | 149 +++++++----------- 1 file changed, 54 insertions(+), 95 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java index c2323ed0f3bff..41946bffb1ff5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java @@ -69,45 +69,10 @@ public void testUnFollow() { IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); Mockito.when(adminClient.indices()).thenReturn(indicesClient); - // Mock pause follow api call: - Mockito.doAnswer(invocation -> { - PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowIndex(), equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); - - // Mock close index api call: - Mockito.doAnswer(invocation -> { - CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; - assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(indicesClient).close(Mockito.any(), Mockito.any()); - - // Mock unfollow api call: - Mockito.doAnswer(invocation -> { - UnfollowAction.Request request = (UnfollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowerIndex(), equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); - - // Mock open index api call: - Mockito.doAnswer(invocation -> { - OpenIndexRequest closeIndexRequest = (OpenIndexRequest) invocation.getArguments()[0]; - assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onResponse(new OpenIndexResponse(true, true)); - return null; - }).when(indicesClient).open(Mockito.any(), Mockito.any()); + mockPauseApiCall(client); + mockCloseIndexApiCall(indicesClient); + mockUnfollowApiCall(client); + mockOpenIndexApiCall(indicesClient); Boolean[] completed = new Boolean[1]; Exception[] failure = new Exception[1]; @@ -141,35 +106,9 @@ public void testUnFollowOpenIndexFailed() { IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); Mockito.when(adminClient.indices()).thenReturn(indicesClient); - // Mock pause follow api call: - Mockito.doAnswer(invocation -> { - PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowIndex(), equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); - - // Mock close index api call: - Mockito.doAnswer(invocation -> { - CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; - assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(indicesClient).close(Mockito.any(), Mockito.any()); - - // Mock unfollow api call: - Mockito.doAnswer(invocation -> { - UnfollowAction.Request request = (UnfollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowerIndex(), equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); + mockPauseApiCall(client); + mockCloseIndexApiCall(indicesClient); + mockUnfollowApiCall(client); // Fail open index api call: Exception error = new RuntimeException(); @@ -213,25 +152,9 @@ public void testUnFollowUnfollowFailed() { IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); Mockito.when(adminClient.indices()).thenReturn(indicesClient); - // Mock pause follow api call: - Mockito.doAnswer(invocation -> { - PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowIndex(), equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + mockPauseApiCall(client); - // Mock close index api call: - Mockito.doAnswer(invocation -> { - CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; - assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(indicesClient).close(Mockito.any(), Mockito.any()); + mockCloseIndexApiCall(indicesClient); // Mock unfollow api call: Exception error = new RuntimeException(); @@ -275,15 +198,7 @@ public void testUnFollowCloseIndexFailed() { IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); Mockito.when(adminClient.indices()).thenReturn(indicesClient); - // Mock pause follow api call: - Mockito.doAnswer(invocation -> { - PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowIndex(), equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + mockPauseApiCall(client); // Mock close index api call: Exception error = new RuntimeException(); @@ -379,4 +294,48 @@ public void onFailure(Exception e) { assertThat(failure[0], nullValue()); Mockito.verifyZeroInteractions(client); } + + private void mockOpenIndexApiCall(IndicesAdminClient indicesClient) { + Mockito.doAnswer(invocation -> { + OpenIndexRequest closeIndexRequest = (OpenIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new OpenIndexResponse(true, true)); + return null; + }).when(indicesClient).open(Mockito.any(), Mockito.any()); + } + + private void mockUnfollowApiCall(Client client) { + Mockito.doAnswer(invocation -> { + UnfollowAction.Request request = (UnfollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowerIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); + } + + private void mockCloseIndexApiCall(IndicesAdminClient indicesClient) { + Mockito.doAnswer(invocation -> { + CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(indicesClient).close(Mockito.any(), Mockito.any()); + } + + private void mockPauseApiCall(Client client) { + Mockito.doAnswer(invocation -> { + PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + } } From 37bd0ef880d2896f968880a2be026b04699c0379 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 4 Jan 2019 09:42:05 -0700 Subject: [PATCH 09/35] Fix error message in test --- .../core/indexlifecycle/WaitForIndexingCompleteStepTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java index 7654a6660bcf1..d7526fcbb18ed 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java @@ -106,6 +106,7 @@ public void testConditionNotMet() { assertThat(result.getInfomationContext(), notNullValue()); WaitForIndexingCompleteStep.Info info = (WaitForIndexingCompleteStep.Info) result.getInfomationContext(); assertThat(info.getIndexSettings(), equalTo(indexMetadata.getSettings())); - assertThat(info.getMessage(), equalTo("the [index.lifecycle.indexing_complete] setting has not been set to true")); + assertThat(info.getMessage(), equalTo("the [index.lifecycle.indexing_complete] setting has not been set to " + + "true on the leader index")); } } From 94dc9519c179618aaa137137c1f7f050f6303f9a Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 4 Jan 2019 17:55:36 -0700 Subject: [PATCH 10/35] [WIP] Add rollover-based test case --- .../indexlifecycle/CCRIndexLifecycleIT.java | 137 ++++++++++++++++-- 1 file changed, 124 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java index 7f9e9a96e4354..0c5aaef11c1fd 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java @@ -12,11 +12,13 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.ccr.ESCCRRestTestCase; import java.io.IOException; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; @@ -30,20 +32,21 @@ public class CCRIndexLifecycleIT extends ESCCRRestTestCase { public void testBasicCCRAndILMIntegration() throws Exception { String indexName = "logs-1"; + String policyName = "basic-test"; if ("leader".equals(targetCluster)) { - putILMPolicy(); + putILMPolicy(policyName, "50GB", null, TimeValue.timeValueHours(7*24)); Settings indexSettings = Settings.builder() .put("index.soft_deletes.enabled", true) .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) - .put("index.lifecycle.name", "my_policy") + .put("index.lifecycle.name", policyName) .put("index.lifecycle.rollover_alias", "logs") .build(); createIndex(indexName, indexSettings, "", "\"logs\": { }"); ensureGreen(indexName); } else if ("follow".equals(targetCluster)) { // Policy with the same name must exist in follower cluster too: - putILMPolicy(); + putILMPolicy(policyName, "50GB", null, TimeValue.timeValueHours(7*24)); followIndex(indexName, indexName); // Aliases are not copied from leader index, so we need to add that for the rollover action in follower cluster: client().performRequest(new Request("PUT", "/" + indexName + "/_alias/logs")); @@ -57,8 +60,8 @@ public void testBasicCCRAndILMIntegration() throws Exception { // Sanity check that following_index setting has been set, so that we can verify later that this setting has been unset: assertThat(getIndexSetting(client(), indexName, "index.xpack.ccr.following_index"), equalTo("true")); - assertILMPolicy(leaderClient, indexName, "hot"); - assertILMPolicy(client(), indexName, "hot"); + assertILMPolicy(leaderClient, indexName, policyName, "hot"); + assertILMPolicy(client(), indexName, policyName, "hot"); }); updateIndexSettings(leaderClient, indexName, Settings.builder() @@ -71,8 +74,8 @@ public void testBasicCCRAndILMIntegration() throws Exception { assertThat(getIndexSetting(leaderClient, indexName, "index.lifecycle.indexing_complete"), equalTo("true")); assertThat(getIndexSetting(client(), indexName, "index.lifecycle.indexing_complete"), equalTo("true")); - assertILMPolicy(leaderClient, indexName, "warm"); - assertILMPolicy(client(), indexName, "warm"); + assertILMPolicy(leaderClient, indexName, policyName, "warm"); + assertILMPolicy(client(), indexName, policyName, "warm"); // ILM should have placed both indices in the warm phase and there these indices are read-only: assertThat(getIndexSetting(leaderClient, indexName, "index.blocks.write"), equalTo("true")); @@ -87,8 +90,109 @@ public void testBasicCCRAndILMIntegration() throws Exception { } } - private static void putILMPolicy() throws IOException { - final Request request = new Request("PUT", "_ilm/policy/my_policy"); + public void testCcrAndIlmWithRollover() throws Exception { + String alias = "metrics"; + String indexName = "metrics-000001"; + String nextIndexName = "metrics-000002"; + String policyName = "rollover-test"; + + if ("leader".equals(targetCluster)) { + // Create a policy on the leader + putILMPolicy(policyName, null, 1, null); + Request templateRequest = new Request("PUT", "_template/my_template"); + Settings indexSettings = Settings.builder() + .put("index.soft_deletes.enabled", true) + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.lifecycle.name", policyName) + .put("index.lifecycle.rollover_alias", alias) + .build(); + templateRequest.setJsonEntity("{\"index_patterns\": [\"metrics-*\"], \"settings\": " + Strings.toString(indexSettings) + "}"); + assertOK(client().performRequest(templateRequest)); + } else if ("follow".equals(targetCluster)) { + // Policy with the same name must exist in follower cluster too: + putILMPolicy(policyName, null, 1, null); + + // Set up an auto-follow pattern + Request createAutoFollowRequest = new Request("PUT", "/_ccr/auto_follow/my_auto_follow_pattern"); + createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], \"remote_cluster\": \"leader_cluster\"}"); + assertOK(client().performRequest(createAutoFollowRequest)); + + try (RestClient leaderClient = buildLeaderClient()) { + // Create an index on the leader using the template set up above + Request createIndexRequest = new Request("PUT", "/" + indexName); + createIndexRequest.setJsonEntity("{" + + "\"mappings\": {\"_doc\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}, " + + "\"aliases\": {\"" + alias + "\": {\"is_write_index\": true}} }"); + assertOK(leaderClient.performRequest(createIndexRequest)); + // Check that the new index is creeg + Request checkIndexRequest = new Request("GET", "/_cluster/health/" + indexName); + checkIndexRequest.addParameter("wait_for_status", "green"); + checkIndexRequest.addParameter("timeout", "70s"); + checkIndexRequest.addParameter("level", "shards"); + assertOK(leaderClient.performRequest(checkIndexRequest)); + + // Check that it got replicated to the follower + assertBusy(() -> assertTrue(indexExists(indexName))); + + // Aliases are not copied from leader index, so we need to add that for the rollover action in follower cluster: + client().performRequest(new Request("PUT", "/" + indexName + "/_alias/" + alias)); + + index(leaderClient, indexName, "1"); + assertDocumentExists(leaderClient, indexName, "1"); + + assertBusy(() -> { + assertDocumentExists(client(), indexName, "1"); + // Sanity check that following_index setting has been set, so that we can verify later that this setting has been unset: + assertThat(getIndexSetting(client(), indexName, "index.xpack.ccr.following_index"), equalTo("true")); + }); + + // Wait for the index to roll over on the leader + assertBusy(() -> { + assertOK(leaderClient.performRequest(new Request("HEAD", "/" + nextIndexName))); + assertThat(getIndexSetting(leaderClient, indexName, "index.lifecycle.indexing_complete"), equalTo("true")); + + }); + + assertBusy(() -> { + // Wait for the next index should have been created on the leader + assertOK(leaderClient.performRequest(new Request("HEAD", "/" + nextIndexName))); + // And the old index should have a write block and indexing complete set + assertThat(getIndexSetting(leaderClient, indexName, "index.blocks.write"), equalTo("true")); + assertThat(getIndexSetting(leaderClient, indexName, "index.lifecycle.indexing_complete"), equalTo("true")); + + }); + + assertBusy(() -> { + // Wait for the setting to get replicated to the follower + assertThat(getIndexSetting(client(), indexName, "index.lifecycle.indexing_complete"), equalTo("true")); + }, 2, TimeUnit.MINUTES); + // TODO: ^^ I don't know why but this takes like 70 seconds to happen. It doesn't seem like it should and this should + // definitely be fixed before merging. + + assertBusy(() -> { + // ILM should have unfollowed the follower index, so the following_index setting should have been removed: + // (this controls whether the follow engine is used) + assertThat(getIndexSetting(client(), indexName, "index.xpack.ccr.following_index"), nullValue()); + // The next index should have been created on the follower as well + indexExists(nextIndexName); + }); + + assertBusy(() -> { + // And the previously-follower index should be in the warm phase + assertILMPolicy(client(), indexName, policyName, "warm"); + }); + + // Clean up + leaderClient.performRequest(new Request("DELETE", "/_template/my_template")); + } + } else { + fail("unexpected target cluster [" + targetCluster + "]"); + } + } + + private static void putILMPolicy(String name, String maxSize, Integer maxDocs, TimeValue maxAge) throws IOException { + final Request request = new Request("PUT", "_ilm/policy/" + name); XContentBuilder builder = jsonBuilder(); builder.startObject(); { @@ -101,8 +205,15 @@ private static void putILMPolicy() throws IOException { builder.startObject("actions"); { builder.startObject("rollover"); - builder.field("max_size", "50GB"); - builder.field("max_age", "7d"); + if (maxSize != null) { + builder.field("max_size", maxSize); + } + if (maxAge != null) { + builder.field("max_age", maxAge); + } + if (maxDocs != null) { + builder.field("max_docs", maxDocs); + } builder.endObject(); } { @@ -143,13 +254,13 @@ private static void putILMPolicy() throws IOException { assertOK(client().performRequest(request)); } - private static void assertILMPolicy(RestClient client, String index, String expectedPhase) throws IOException { + private static void assertILMPolicy(RestClient client, String index, String policy, String expectedPhase) throws IOException { final Request request = new Request("GET", "/" + index + "/_ilm/explain"); Map response = toMap(client.performRequest(request)); LOGGER.info("response={}", response); Map explanation = (Map) ((Map) response.get("indices")).get(index); assertThat(explanation.get("managed"), is(true)); - assertThat(explanation.get("policy"), equalTo("my_policy")); + assertThat(explanation.get("policy"), equalTo(policy)); assertThat(explanation.get("phase"), equalTo(expectedPhase)); } From 394944510484d31521f51333d3f31f5dc3fde650 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 7 Jan 2019 13:29:33 +0100 Subject: [PATCH 11/35] Decreased read_poll_timeout from 60 seconds to 1 second. The shard follow tasks continuesly poll changes from leader index, in case no new writes arrive in the leader shards, then the leader shards wait up to the defined read_poll_timeout before returning an empty result. The result also includes information about whether the index settings need to be synced. Because in this test no data was indexed it took a full minute for ccr to realize that the followe index settings needed to be updated and caused the test to fail due to an assert busy statement to time out. --- .../xpack/indexlifecycle/CCRIndexLifecycleIT.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java index 0c5aaef11c1fd..797916c7c405f 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.util.Map; -import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; @@ -115,7 +114,8 @@ public void testCcrAndIlmWithRollover() throws Exception { // Set up an auto-follow pattern Request createAutoFollowRequest = new Request("PUT", "/_ccr/auto_follow/my_auto_follow_pattern"); - createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], \"remote_cluster\": \"leader_cluster\"}"); + createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], " + + "\"remote_cluster\": \"leader_cluster\", \"read_poll_timeout\": \"1000ms\"}"); assertOK(client().performRequest(createAutoFollowRequest)); try (RestClient leaderClient = buildLeaderClient()) { @@ -166,9 +166,7 @@ public void testCcrAndIlmWithRollover() throws Exception { assertBusy(() -> { // Wait for the setting to get replicated to the follower assertThat(getIndexSetting(client(), indexName, "index.lifecycle.indexing_complete"), equalTo("true")); - }, 2, TimeUnit.MINUTES); - // TODO: ^^ I don't know why but this takes like 70 seconds to happen. It doesn't seem like it should and this should - // definitely be fixed before merging. + }); assertBusy(() -> { // ILM should have unfollowed the follower index, so the following_index setting should have been removed: From 0ef70023b60e1e6986fdf4db0331f1df4f0c3682 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 7 Jan 2019 14:30:45 +0100 Subject: [PATCH 12/35] Adjusted testGetNextActionName() test --- .../core/indexlifecycle/TimeseriesLifecycleTypeTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java index 784c91cdbdd97..a5ec48994aea8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java @@ -304,6 +304,10 @@ public void testGetNextActionName() { // Hot Phase assertNextActionName("hot", RolloverAction.NAME, null, new String[] {}); assertNextActionName("hot", RolloverAction.NAME, null, new String[] { RolloverAction.NAME }); + assertNextActionName("hot", UnfollowAction.NAME, null, new String[] {}); + assertNextActionName("hot", UnfollowAction.NAME, null, new String[] {UnfollowAction.NAME}); + assertNextActionName("hot", UnfollowAction.NAME, RolloverAction.NAME, + new String[] {UnfollowAction.NAME, RolloverAction.NAME}); assertInvalidAction("hot", "foo", new String[] { RolloverAction.NAME }); assertInvalidAction("hot", AllocateAction.NAME, new String[] { RolloverAction.NAME }); assertInvalidAction("hot", DeleteAction.NAME, new String[] { RolloverAction.NAME }); From 2d368aba15f9287b02da88407d939fbe5fa11bba Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Jan 2019 14:22:10 -0700 Subject: [PATCH 13/35] More informative message when following is paused --- .../core/indexlifecycle/WaitForFollowShardTasksStep.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java index 2df8ad07538ef..f3938a1d3da2b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStep.java @@ -85,7 +85,12 @@ List getShardFollowTaskInfos() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(SHARD_FOLLOW_TASKS.getPreferredName(), shardFollowTaskInfos); - String message = "Waiting for [" + shardFollowTaskInfos.size() + "] shard follow tasks to be in sync"; + String message; + if (shardFollowTaskInfos.size() > 0) { + message = "Waiting for [" + shardFollowTaskInfos.size() + "] shard follow tasks to be in sync"; + } else { + message = "Waiting for following to be unpaused and all shard follow tasks to be up to date"; + } builder.field(MESSAGE.getPreferredName(), message); builder.endObject(); return builder; From 748577334d51b967ea38607093041df5764687f7 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Jan 2019 16:02:12 -0700 Subject: [PATCH 14/35] Remove index settings from step info --- .../WaitForIndexingCompleteStep.java | 19 ++---- .../WaitForIndexingCompleteStepInfoTests.java | 62 ------------------- .../WaitForIndexingCompleteStepTests.java | 1 - 3 files changed, 4 insertions(+), 78 deletions(-) delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepInfoTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java index bc52842d3ead9..2581b0853fafe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java @@ -8,7 +8,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.Index; @@ -39,39 +38,29 @@ public Result isConditionMet(Index index, ClusterState clusterState) { if (indexingComplete) { return new Result(true, null); } else { - return new Result(false, new Info(followerIndex.getSettings())); + return new Result(false, new Info()); } } static final class Info implements ToXContentObject { static final ParseField MESSAGE_FIELD = new ParseField("message"); - static final ParseField INDEX_SETTINGS_FIELD = new ParseField("index_settings"); private final String message; - private final Settings indexSettings; - Info(Settings indexSettings) { + Info() { this.message = "the [" + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE + "] setting has not been set to true on the leader index"; - this.indexSettings = indexSettings; } String getMessage() { return message; } - Settings getIndexSettings() { - return indexSettings; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(MESSAGE_FIELD.getPreferredName(), message); - builder.startObject(INDEX_SETTINGS_FIELD.getPreferredName()); - indexSettings.toXContent(builder, params); - builder.endObject(); builder.endObject(); return builder; } @@ -81,12 +70,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Info info = (Info) o; - return Objects.equals(indexSettings, info.indexSettings); + return Objects.equals(getMessage(), info.getMessage()); } @Override public int hashCode() { - return Objects.hash(indexSettings); + return Objects.hash(getMessage()); } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepInfoTests.java deleted file mode 100644 index dcd29fb3b10e2..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepInfoTests.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.core.indexlifecycle; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.AbstractXContentTestCase; -import org.elasticsearch.test.EqualsHashCodeTestUtils; -import org.elasticsearch.xpack.core.indexlifecycle.WaitForIndexingCompleteStep.Info; - -import java.io.IOException; - -import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings; - -public class WaitForIndexingCompleteStepInfoTests extends AbstractXContentTestCase { - - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("wait_for_indexing_complete_info", - args -> new Info((Settings) args[0])); - static { - PARSER.declareObject(ConstructingObjectParser.constructorArg(), - (p, c) -> Settings.fromXContent(p), Info.INDEX_SETTINGS_FIELD); - PARSER.declareString((i, s) -> {}, Info.MESSAGE_FIELD); - } - - @Override - protected Info createTestInstance() { - return new Info(randomIndexSettings()); - } - - @Override - protected Info doParseInstance(XContentParser parser) throws IOException { - return PARSER.apply(parser, null); - } - - @Override - protected boolean supportsUnknownFields() { - return false; - } - - public final void testEqualsAndHashcode() { - for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { - EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copyInstance, this::mutateInstance); - } - } - - protected final Info copyInstance(Info instance) throws IOException { - return new Info(instance.getIndexSettings()); - } - - protected Info mutateInstance(Info instance) throws IOException { - Settings.Builder newSettings = Settings.builder().put(instance.getIndexSettings()); - newSettings.put(randomAlphaOfLength(4), randomAlphaOfLength(4)); - return new Info(newSettings.build()); - } - -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java index d7526fcbb18ed..61fee0c4178a1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java @@ -105,7 +105,6 @@ public void testConditionNotMet() { assertThat(result.isComplete(), is(false)); assertThat(result.getInfomationContext(), notNullValue()); WaitForIndexingCompleteStep.Info info = (WaitForIndexingCompleteStep.Info) result.getInfomationContext(); - assertThat(info.getIndexSettings(), equalTo(indexMetadata.getSettings())); assertThat(info.getMessage(), equalTo("the [index.lifecycle.indexing_complete] setting has not been set to " + "true on the leader index")); } From d96ebbbeb811488f94b6864645ab1bbb1fb94f0b Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Jan 2019 16:21:14 -0700 Subject: [PATCH 15/35] Javadoc for UnfollowAction --- .../xpack/core/indexlifecycle/UnfollowAction.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java index 0d333b001a7bb..1faeef6c589ff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java @@ -18,6 +18,15 @@ import java.util.Arrays; import java.util.List; +/** + * Converts a CCR following index into a normal, standalone index, once the index is ready to be safely separated. + * + * "Readiness" is composed of two conditions: + * 1) The index must have {@link LifecycleSettings#LIFECYCLE_INDEXING_COMPLETE} set to {@code true}, which is + * done automatically by {@link RolloverAction} (or manually). + * 2) The index must be up to date with the leader, defined as the follower checkpoint being + * equal to the global checkpoint for all shards. + */ public final class UnfollowAction implements LifecycleAction { public static final String NAME = "unfollow"; From 464c25665aa22e2d04a9ecddebc2bf948c4999cb Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Jan 2019 16:22:33 -0700 Subject: [PATCH 16/35] UnfollowAction hashcode --- .../elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java index 1faeef6c589ff..f6d6090b74bfb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java @@ -85,7 +85,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { - return 1; + return 36970; } @Override From 5d63a5aab22a411746ded5c516d3485805785580 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Jan 2019 16:40:34 -0700 Subject: [PATCH 17/35] Handle deleted indices in WaitForIndexingComplete --- .../indexlifecycle/WaitForIndexingCompleteStep.java | 10 +++++++++- .../WaitForIndexingCompleteStepTests.java | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java index 2581b0853fafe..db3ede990ab9c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.indexlifecycle; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseField; @@ -19,6 +21,7 @@ import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; final class WaitForIndexingCompleteStep extends ClusterStateWaitStep { + private static final Logger logger = LogManager.getLogger(WaitForIndexingCompleteStep.class); static final String NAME = "wait-for-indexing-complete"; @@ -28,7 +31,12 @@ final class WaitForIndexingCompleteStep extends ClusterStateWaitStep { @Override public Result isConditionMet(Index index, ClusterState clusterState) { - IndexMetaData followerIndex = clusterState.metaData().getIndexSafe(index); + IndexMetaData followerIndex = clusterState.metaData().index(index); + if (followerIndex == null) { + // Index must have been since deleted, ignore it + logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName()); + return new Result(false, null); + } Map customIndexMetadata = followerIndex.getCustomData(CCR_METADATA_KEY); if (customIndexMetadata == null) { return new Result(true, null); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java index 61fee0c4178a1..4984e33eee0ab 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; import java.util.Collections; @@ -108,4 +109,15 @@ public void testConditionNotMet() { assertThat(info.getMessage(), equalTo("the [index.lifecycle.indexing_complete] setting has not been set to " + "true on the leader index")); } + + public void testIndexDeleted() { + ClusterState clusterState = ClusterState.builder(new ClusterName("cluster")) + .metaData(MetaData.builder().build()) + .build(); + + WaitForIndexingCompleteStep step = createRandomInstance(); + ClusterStateWaitStep.Result result = step.isConditionMet(new Index("this-index-doesnt-exist", "uuid"), clusterState); + assertThat(result.isComplete(), is(false)); + assertThat(result.getInfomationContext(), nullValue()); + } } From f63e73d8389f04c1200112b06a7b32899beeef67 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 9 Jan 2019 13:37:26 +0100 Subject: [PATCH 18/35] added assertion messages --- .../core/indexlifecycle/UnfollowFollowIndexStep.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java index 42ee2353f9496..5fbb66100daba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java @@ -42,7 +42,7 @@ void pauseFollowerIndex(final String followerIndex, final Listener listener) { PauseFollowAction.Request request = new PauseFollowAction.Request(followerIndex); getClient().execute(PauseFollowAction.INSTANCE, request, ActionListener.wrap( r -> { - assert r.isAcknowledged(); + assert r.isAcknowledged() : "pause follow response is not acknowledge"; closeFollowerIndex(followerIndex, listener); }, listener::onFailure @@ -52,8 +52,8 @@ void pauseFollowerIndex(final String followerIndex, final Listener listener) { void closeFollowerIndex(final String followerIndex, final Listener listener) { CloseIndexRequest closeIndexRequest = new CloseIndexRequest(followerIndex); getClient().admin().indices().close(closeIndexRequest, ActionListener.wrap( - acknowledgedResponse -> { - assert acknowledgedResponse.isAcknowledged(); + r -> { + assert r.isAcknowledged() : "close index response is not acknowledge"; unfollow(followerIndex, listener); }, listener::onFailure) @@ -64,7 +64,7 @@ void unfollow(final String followerIndex, final Listener listener) { UnfollowAction.Request request = new UnfollowAction.Request(followerIndex); getClient().execute(UnfollowAction.INSTANCE, request, ActionListener.wrap( r -> { - assert r.isAcknowledged(); + assert r.isAcknowledged() : "unfollow response is not acknowledge"; openIndex(followerIndex, listener); }, listener::onFailure @@ -74,8 +74,8 @@ void unfollow(final String followerIndex, final Listener listener) { void openIndex(final String index, final Listener listener) { OpenIndexRequest request = new OpenIndexRequest(index); getClient().admin().indices().open(request, ActionListener.wrap( - openIndexResponse -> { - assert openIndexResponse.isAcknowledged(); + r -> { + assert r.isAcknowledged() : "open index response is not acknowledge"; listener.onResponse(true); }, listener::onFailure From 886a3f55badcf449a40195c6a11ebeb9a1be09e1 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 9 Jan 2019 19:14:21 +0100 Subject: [PATCH 19/35] Moved pause follower index, close follower index and open follower index operation from UnfollowFollowerIndexStep to the following steps respectively: * PauseFollowerIndexStep * CloseFollowerIndexStep * OpenFollowerIndexStep The UnfollowAction now exists of the following steps: * WaitForIndexingCompleteStep * WaitForFollowShardStep * PauseFollowerIndexStep * CloseFollowerIndexStep * UnfollowFollowerIndexStep * OpenFollowerIndexStep --- .../AbstractUnfollowIndexStep.java | 35 +++ .../CloseFollowerIndexStep.java | 32 +++ .../indexlifecycle/OpenFollowerIndexStep.java | 31 +++ .../PauseFollowerIndexStep.java | 31 +++ .../core/indexlifecycle/UnfollowAction.java | 20 +- .../UnfollowFollowIndexStep.java | 61 +---- .../AbstractUnfollowIndexStepTestCase.java | 73 ++++++ .../CloseFollowerIndexStepTests.java | 118 +++++++++ .../OpenFollowerIndexStepTests.java | 116 ++++++++ .../PauseFollowerIndexStepTests.java | 112 ++++++++ .../indexlifecycle/UnfollowActionTests.java | 23 +- .../UnfollowFollowIndexStepTests.java | 248 +----------------- 12 files changed, 597 insertions(+), 303 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractUnfollowIndexStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStep.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PauseFollowerIndexStep.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractUnfollowIndexStepTestCase.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStepTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PauseFollowerIndexStepTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractUnfollowIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractUnfollowIndexStep.java new file mode 100644 index 0000000000000..8e0626425b490 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractUnfollowIndexStep.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; + +import java.util.Map; + +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; + +abstract class AbstractUnfollowIndexStep extends AsyncActionStep { + + AbstractUnfollowIndexStep(StepKey key, StepKey nextStepKey, Client client) { + super(key, nextStepKey, client); + } + + @Override + public final void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener) { + String followerIndex = indexMetaData.getIndex().getName(); + Map customIndexMetadata = indexMetaData.getCustomData(CCR_METADATA_KEY); + if (customIndexMetadata == null) { + listener.onResponse(true); + return; + } + + innerPerformAction(followerIndex, listener); + } + + abstract void innerPerformAction(String followerIndex, Listener listener); +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStep.java new file mode 100644 index 0000000000000..3c6c3f4c04505 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStep.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; + +final class CloseFollowerIndexStep extends AbstractUnfollowIndexStep { + + static final String NAME = "close-follower-index"; + + CloseFollowerIndexStep(StepKey key, StepKey nextStepKey, Client client) { + super(key, nextStepKey, client); + } + + @Override + void innerPerformAction(String followerIndex, Listener listener) { + CloseIndexRequest closeIndexRequest = new CloseIndexRequest(followerIndex); + getClient().admin().indices().close(closeIndexRequest, ActionListener.wrap( + r -> { + assert r.isAcknowledged() : "close index response is not acknowledged"; + listener.onResponse(true); + }, + listener::onFailure) + ); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStep.java new file mode 100644 index 0000000000000..a95fa1d97cc9f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStep.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.client.Client; + +final class OpenFollowerIndexStep extends AbstractUnfollowIndexStep { + + static final String NAME = "open-follower-index"; + + OpenFollowerIndexStep(StepKey key, StepKey nextStepKey, Client client) { + super(key, nextStepKey, client); + } + + @Override + void innerPerformAction(String followerIndex, Listener listener) { + OpenIndexRequest request = new OpenIndexRequest(followerIndex); + getClient().admin().indices().open(request, ActionListener.wrap( + r -> { + assert r.isAcknowledged() : "open index response is not acknowledged"; + listener.onResponse(true); + }, + listener::onFailure + )); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PauseFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PauseFollowerIndexStep.java new file mode 100644 index 0000000000000..72b38c7b72797 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PauseFollowerIndexStep.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; + +final class PauseFollowerIndexStep extends AbstractUnfollowIndexStep { + + static final String NAME = "pause-follower-index"; + + PauseFollowerIndexStep(StepKey key, StepKey nextStepKey, Client client) { + super(key, nextStepKey, client); + } + + @Override + void innerPerformAction(String followerIndex, Listener listener) { + PauseFollowAction.Request request = new PauseFollowAction.Request(followerIndex); + getClient().execute(PauseFollowAction.INSTANCE, request, ActionListener.wrap( + r -> { + assert r.isAcknowledged() : "pause follow response is not acknowledged"; + listener.onResponse(true); + }, + listener::onFailure + )); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java index f6d6090b74bfb..b70d5e8441ef8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java @@ -38,20 +38,30 @@ public UnfollowAction() {} public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey indexingComplete = new StepKey(phase, NAME, WaitForIndexingCompleteStep.NAME); StepKey waitForFollowShardTasks = new StepKey(phase, NAME, WaitForFollowShardTasksStep.NAME); - StepKey unfollowIndex = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); + StepKey pauseFollowerIndex = new StepKey(phase, NAME, PauseFollowerIndexStep.NAME); + StepKey closeFollowerIndex = new StepKey(phase, NAME, CloseFollowerIndexStep.NAME); + StepKey unfollowFollowerIndex = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); + StepKey openFollowerIndex = new StepKey(phase, NAME, OpenFollowerIndexStep.NAME); WaitForIndexingCompleteStep step1 = new WaitForIndexingCompleteStep(indexingComplete, waitForFollowShardTasks); - WaitForFollowShardTasksStep step2 = new WaitForFollowShardTasksStep(waitForFollowShardTasks, unfollowIndex, client); - UnfollowFollowIndexStep step3 = new UnfollowFollowIndexStep(unfollowIndex, nextStepKey, client); - return Arrays.asList(step1, step2, step3); + WaitForFollowShardTasksStep step2 = new WaitForFollowShardTasksStep(waitForFollowShardTasks, pauseFollowerIndex, client); + PauseFollowerIndexStep step3 = new PauseFollowerIndexStep(pauseFollowerIndex, closeFollowerIndex, client); + CloseFollowerIndexStep step4 = new CloseFollowerIndexStep(closeFollowerIndex, unfollowFollowerIndex, client); + UnfollowFollowIndexStep step5 = new UnfollowFollowIndexStep(unfollowFollowerIndex, openFollowerIndex, client); + OpenFollowerIndexStep step6 = new OpenFollowerIndexStep(openFollowerIndex, nextStepKey, client); + return Arrays.asList(step1, step2, step3, step4, step5, step6); } @Override public List toStepKeys(String phase) { StepKey indexingCompleteStep = new StepKey(phase, NAME, WaitForIndexingCompleteStep.NAME); StepKey waitForFollowShardTasksStep = new StepKey(phase, NAME, WaitForFollowShardTasksStep.NAME); + StepKey pauseFollowerIndexStep = new StepKey(phase, NAME, PauseFollowerIndexStep.NAME); + StepKey closeFollowerIndexStep = new StepKey(phase, NAME, CloseFollowerIndexStep.NAME); StepKey unfollowIndexStep = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); - return Arrays.asList(indexingCompleteStep, waitForFollowShardTasksStep, unfollowIndexStep); + StepKey openFollowerIndexStep = new StepKey(phase, NAME, OpenFollowerIndexStep.NAME); + return Arrays.asList(indexingCompleteStep, waitForFollowShardTasksStep, pauseFollowerIndexStep, + closeFollowerIndexStep, unfollowIndexStep, openFollowerIndexStep); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java index 5fbb66100daba..953450bbc763b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStep.java @@ -6,76 +6,23 @@ package org.elasticsearch.xpack.core.indexlifecycle; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; import org.elasticsearch.xpack.core.ccr.action.UnfollowAction; -import java.util.Map; +final class UnfollowFollowIndexStep extends AbstractUnfollowIndexStep { -import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; - -final class UnfollowFollowIndexStep extends AsyncActionStep { - - static final String NAME = "unfollow-index"; + static final String NAME = "unfollow-follower-index"; UnfollowFollowIndexStep(StepKey key, StepKey nextStepKey, Client client) { super(key, nextStepKey, client); } @Override - public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener) { - String followerIndex = indexMetaData.getIndex().getName(); - Map customIndexMetadata = indexMetaData.getCustomData(CCR_METADATA_KEY); - if (customIndexMetadata == null) { - listener.onResponse(true); - return; - } - - pauseFollowerIndex(followerIndex, listener); - } - - void pauseFollowerIndex(final String followerIndex, final Listener listener) { - PauseFollowAction.Request request = new PauseFollowAction.Request(followerIndex); - getClient().execute(PauseFollowAction.INSTANCE, request, ActionListener.wrap( - r -> { - assert r.isAcknowledged() : "pause follow response is not acknowledge"; - closeFollowerIndex(followerIndex, listener); - }, - listener::onFailure - )); - } - - void closeFollowerIndex(final String followerIndex, final Listener listener) { - CloseIndexRequest closeIndexRequest = new CloseIndexRequest(followerIndex); - getClient().admin().indices().close(closeIndexRequest, ActionListener.wrap( - r -> { - assert r.isAcknowledged() : "close index response is not acknowledge"; - unfollow(followerIndex, listener); - }, - listener::onFailure) - ); - } - - void unfollow(final String followerIndex, final Listener listener) { + void innerPerformAction(String followerIndex, Listener listener) { UnfollowAction.Request request = new UnfollowAction.Request(followerIndex); getClient().execute(UnfollowAction.INSTANCE, request, ActionListener.wrap( r -> { - assert r.isAcknowledged() : "unfollow response is not acknowledge"; - openIndex(followerIndex, listener); - }, - listener::onFailure - )); - } - - void openIndex(final String index, final Listener listener) { - OpenIndexRequest request = new OpenIndexRequest(index); - getClient().admin().indices().open(request, ActionListener.wrap( - r -> { - assert r.isAcknowledged() : "open index response is not acknowledge"; + assert r.isAcknowledged() : "unfollow response is not acknowledged"; listener.onResponse(true); }, listener::onFailure diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractUnfollowIndexStepTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractUnfollowIndexStepTestCase.java new file mode 100644 index 0000000000000..5ceb8ca657006 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractUnfollowIndexStepTestCase.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.mockito.Mockito; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public abstract class AbstractUnfollowIndexStepTestCase extends AbstractStepTestCase { + + @Override + protected final T createRandomInstance() { + Step.StepKey stepKey = randomStepKey(); + Step.StepKey nextStepKey = randomStepKey(); + return newInstance(stepKey, nextStepKey, Mockito.mock(Client.class)); + } + + @Override + protected final T mutateInstance(T instance) { + Step.StepKey key = instance.getKey(); + Step.StepKey nextKey = instance.getNextStepKey(); + + if (randomBoolean()) { + key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } else { + nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } + + return newInstance(key, nextKey, instance.getClient()); + } + + @Override + protected final T copyInstance(T instance) { + return newInstance(instance.getKey(), instance.getNextStepKey(), instance.getClient()); + } + + public final void testNotAFollowerIndex() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + T step = newInstance(randomStepKey(), randomStepKey(), client); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], is(true)); + assertThat(failure[0], nullValue()); + Mockito.verifyZeroInteractions(client); + } + + protected abstract T newInstance(Step.StepKey key, Step.StepKey nextKey, Client client); +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java new file mode 100644 index 0000000000000..fd0660ec224c4 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.AdminClient; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; +import org.mockito.Mockito; + +import java.util.Collections; + +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; + +public class CloseFollowerIndexStepTests extends AbstractUnfollowIndexStepTestCase { + + @Override + protected CloseFollowerIndexStep newInstance(Step.StepKey key, Step.StepKey nextKey, Client client) { + return new CloseFollowerIndexStep(key, nextKey, client); + } + + public void testCloseFollowingIndex() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + Mockito.doAnswer(invocation -> { + CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(indicesClient).close(Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + CloseFollowerIndexStep step = new CloseFollowerIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], is(true)); + assertThat(failure[0], nullValue()); + } + + public void testCloseFollowingIndexFailed() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + // Mock pause follow api call: + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + Exception error = new RuntimeException(); + Mockito.doAnswer(invocation -> { + CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onFailure(error); + return null; + }).when(indicesClient).close(Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + CloseFollowerIndexStep step = new CloseFollowerIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], nullValue()); + assertThat(failure[0], sameInstance(error)); + Mockito.verify(indicesClient).close(Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(indicesClient); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStepTests.java new file mode 100644 index 0000000000000..bd254fe0245ff --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStepTests.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; +import org.elasticsearch.client.AdminClient; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.mockito.Mockito; + +import java.util.Collections; + +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; + +public class OpenFollowerIndexStepTests extends AbstractUnfollowIndexStepTestCase { + + @Override + protected OpenFollowerIndexStep newInstance(Step.StepKey key, Step.StepKey nextKey, Client client) { + return new OpenFollowerIndexStep(key, nextKey, client); + } + + public void testOpenFollowingIndex() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + Mockito.doAnswer(invocation -> { + OpenIndexRequest closeIndexRequest = (OpenIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onResponse(new OpenIndexResponse(true, true)); + return null; + }).when(indicesClient).open(Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + OpenFollowerIndexStep step = new OpenFollowerIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], is(true)); + assertThat(failure[0], nullValue()); + } + + public void testOpenFollowingIndexFailed() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + Exception error = new RuntimeException(); + Mockito.doAnswer(invocation -> { + OpenIndexRequest closeIndexRequest = (OpenIndexRequest) invocation.getArguments()[0]; + assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); + ActionListener listener = (ActionListener) invocation.getArguments()[1]; + listener.onFailure(error); + return null; + }).when(indicesClient).open(Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + OpenFollowerIndexStep step = new OpenFollowerIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], nullValue()); + assertThat(failure[0], sameInstance(error)); + Mockito.verify(indicesClient).open(Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(indicesClient); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PauseFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PauseFollowerIndexStepTests.java new file mode 100644 index 0000000000000..fa877ef080ff4 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PauseFollowerIndexStepTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.AdminClient; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; +import org.mockito.Mockito; + +import java.util.Collections; + +import static org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction.CCR_METADATA_KEY; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; + +public class PauseFollowerIndexStepTests extends AbstractUnfollowIndexStepTestCase { + + @Override + protected PauseFollowerIndexStep newInstance(Step.StepKey key, Step.StepKey nextKey, Client client) { + return new PauseFollowerIndexStep(key, nextKey, client); + } + + public void testPauseFollowingIndex() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + Client client = Mockito.mock(Client.class); + AdminClient adminClient = Mockito.mock(AdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); + Mockito.when(adminClient.indices()).thenReturn(indicesClient); + + Mockito.doAnswer(invocation -> { + PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); + return null; + }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + PauseFollowerIndexStep step = new PauseFollowerIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], is(true)); + assertThat(failure[0], nullValue()); + } + + public void testPauseFollowingIndexFailed() { + IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") + .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + // Mock pause follow api call: + Client client = Mockito.mock(Client.class); + Exception error = new RuntimeException(); + Mockito.doAnswer(invocation -> { + PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowIndex(), equalTo("follower-index")); + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onFailure(error); + return null; + }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + + Boolean[] completed = new Boolean[1]; + Exception[] failure = new Exception[1]; + PauseFollowerIndexStep step = new PauseFollowerIndexStep(randomStepKey(), randomStepKey(), client); + step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { + @Override + public void onResponse(boolean complete) { + completed[0] = complete; + } + + @Override + public void onFailure(Exception e) { + failure[0] = e; + } + }); + assertThat(completed[0], nullValue()); + assertThat(failure[0], sameInstance(error)); + Mockito.verify(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(client); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java index afe6488da2903..e107b53153f8c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java @@ -39,11 +39,14 @@ public void testToSteps() { randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); assertThat(steps, notNullValue()); - assertThat(steps.size(), equalTo(3)); + assertThat(steps.size(), equalTo(6)); StepKey expectedFirstStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForIndexingCompleteStep.NAME); StepKey expectedSecondStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForFollowShardTasksStep.NAME); - StepKey expectedThirdStepKey = new StepKey(phase, UnfollowAction.NAME, UnfollowFollowIndexStep.NAME); + StepKey expectedThirdStepKey = new StepKey(phase, UnfollowAction.NAME, PauseFollowerIndexStep.NAME); + StepKey expectedFourthStepKey = new StepKey(phase, UnfollowAction.NAME, CloseFollowerIndexStep.NAME); + StepKey expectedFifthStepKey = new StepKey(phase, UnfollowAction.NAME, UnfollowFollowIndexStep.NAME); + StepKey expectedSixthStepKey = new StepKey(phase, UnfollowAction.NAME, OpenFollowerIndexStep.NAME); WaitForIndexingCompleteStep firstStep = (WaitForIndexingCompleteStep) steps.get(0); assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey)); @@ -53,8 +56,20 @@ public void testToSteps() { assertThat(secondStep.getKey(), equalTo(expectedSecondStepKey)); assertThat(secondStep.getNextStepKey(), equalTo(expectedThirdStepKey)); - UnfollowFollowIndexStep thirdStep = (UnfollowFollowIndexStep) steps.get(2); + PauseFollowerIndexStep thirdStep = (PauseFollowerIndexStep) steps.get(2); assertThat(thirdStep.getKey(), equalTo(expectedThirdStepKey)); - assertThat(thirdStep.getNextStepKey(), equalTo(nextStepKey)); + assertThat(thirdStep.getNextStepKey(), equalTo(expectedFourthStepKey)); + + CloseFollowerIndexStep fourthStep = (CloseFollowerIndexStep) steps.get(3); + assertThat(fourthStep.getKey(), equalTo(expectedFourthStepKey)); + assertThat(fourthStep.getNextStepKey(), equalTo(expectedFifthStepKey)); + + UnfollowFollowIndexStep fifthStep = (UnfollowFollowIndexStep) steps.get(4); + assertThat(fifthStep.getKey(), equalTo(expectedFifthStepKey)); + assertThat(fifthStep.getNextStepKey(), equalTo(expectedSixthStepKey)); + + OpenFollowerIndexStep sixthStep = (OpenFollowerIndexStep) steps.get(5); + assertThat(sixthStep.getKey(), equalTo(expectedSixthStepKey)); + assertThat(sixthStep.getNextStepKey(), equalTo(nextStepKey)); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java index 41946bffb1ff5..58558c92d2511 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowFollowIndexStepTests.java @@ -7,15 +7,11 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; -import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; import org.elasticsearch.xpack.core.ccr.action.UnfollowAction; import org.mockito.Mockito; @@ -27,32 +23,11 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -public class UnfollowFollowIndexStepTests extends AbstractStepTestCase { +public class UnfollowFollowIndexStepTests extends AbstractUnfollowIndexStepTestCase { @Override - protected UnfollowFollowIndexStep createRandomInstance() { - Step.StepKey stepKey = randomStepKey(); - Step.StepKey nextStepKey = randomStepKey(); - return new UnfollowFollowIndexStep(stepKey, nextStepKey, Mockito.mock(Client.class)); - } - - @Override - protected UnfollowFollowIndexStep mutateInstance(UnfollowFollowIndexStep instance) { - Step.StepKey key = instance.getKey(); - Step.StepKey nextKey = instance.getNextStepKey(); - - if (randomBoolean()) { - key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); - } else { - nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); - } - - return new UnfollowFollowIndexStep(key, nextKey, instance.getClient()); - } - - @Override - protected UnfollowFollowIndexStep copyInstance(UnfollowFollowIndexStep instance) { - return new UnfollowFollowIndexStep(instance.getKey(), instance.getNextStepKey(), instance.getClient()); + protected UnfollowFollowIndexStep newInstance(Step.StepKey key, Step.StepKey nextKey, Client client) { + return new UnfollowFollowIndexStep(key, nextKey, client); } public void testUnFollow() { @@ -69,56 +44,14 @@ public void testUnFollow() { IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); Mockito.when(adminClient.indices()).thenReturn(indicesClient); - mockPauseApiCall(client); - mockCloseIndexApiCall(indicesClient); - mockUnfollowApiCall(client); - mockOpenIndexApiCall(indicesClient); - - Boolean[] completed = new Boolean[1]; - Exception[] failure = new Exception[1]; - UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); - step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { - @Override - public void onResponse(boolean complete) { - completed[0] = complete; - } - - @Override - public void onFailure(Exception e) { - failure[0] = e; - } - }); - assertThat(completed[0], is(true)); - assertThat(failure[0], nullValue()); - } - - public void testUnFollowOpenIndexFailed() { - IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") - .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - Client client = Mockito.mock(Client.class); - AdminClient adminClient = Mockito.mock(AdminClient.class); - Mockito.when(client.admin()).thenReturn(adminClient); - IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); - Mockito.when(adminClient.indices()).thenReturn(indicesClient); - - mockPauseApiCall(client); - mockCloseIndexApiCall(indicesClient); - mockUnfollowApiCall(client); - - // Fail open index api call: - Exception error = new RuntimeException(); Mockito.doAnswer(invocation -> { - OpenIndexRequest closeIndexRequest = (OpenIndexRequest) invocation.getArguments()[0]; - assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onFailure(error); + UnfollowAction.Request request = (UnfollowAction.Request) invocation.getArguments()[1]; + assertThat(request.getFollowerIndex(), equalTo("follower-index")); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AcknowledgedResponse(true)); return null; - }).when(indicesClient).open(Mockito.any(), Mockito.any()); + }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); Boolean[] completed = new Boolean[1]; Exception[] failure = new Exception[1]; @@ -134,8 +67,8 @@ public void onFailure(Exception e) { failure[0] = e; } }); - assertThat(completed[0], nullValue()); - assertThat(failure[0], sameInstance(error)); + assertThat(completed[0], is(true)); + assertThat(failure[0], nullValue()); } public void testUnFollowUnfollowFailed() { @@ -152,10 +85,6 @@ public void testUnFollowUnfollowFailed() { IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); Mockito.when(adminClient.indices()).thenReturn(indicesClient); - mockPauseApiCall(client); - - mockCloseIndexApiCall(indicesClient); - // Mock unfollow api call: Exception error = new RuntimeException(); Mockito.doAnswer(invocation -> { @@ -183,159 +112,4 @@ public void onFailure(Exception e) { assertThat(completed[0], nullValue()); assertThat(failure[0], sameInstance(error)); } - - public void testUnFollowCloseIndexFailed() { - IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") - .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - Client client = Mockito.mock(Client.class); - AdminClient adminClient = Mockito.mock(AdminClient.class); - Mockito.when(client.admin()).thenReturn(adminClient); - IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class); - Mockito.when(adminClient.indices()).thenReturn(indicesClient); - - mockPauseApiCall(client); - - // Mock close index api call: - Exception error = new RuntimeException(); - Mockito.doAnswer(invocation -> { - CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; - assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onFailure(error); - return null; - }).when(indicesClient).close(Mockito.any(), Mockito.any()); - - Boolean[] completed = new Boolean[1]; - Exception[] failure = new Exception[1]; - UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); - step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { - @Override - public void onResponse(boolean complete) { - completed[0] = complete; - } - - @Override - public void onFailure(Exception e) { - failure[0] = e; - } - }); - assertThat(completed[0], nullValue()); - assertThat(failure[0], sameInstance(error)); - } - - public void testUnFollowPauseIndexFollowingFailed() { - IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") - .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .putCustom(CCR_METADATA_KEY, Collections.emptyMap()) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - // Mock pause follow api call: - Client client = Mockito.mock(Client.class); - Exception error = new RuntimeException(); - Mockito.doAnswer(invocation -> { - PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowIndex(), equalTo("follower-index")); - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onFailure(error); - return null; - }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); - - Boolean[] completed = new Boolean[1]; - Exception[] failure = new Exception[1]; - UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); - step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { - @Override - public void onResponse(boolean complete) { - completed[0] = complete; - } - - @Override - public void onFailure(Exception e) { - failure[0] = e; - } - }); - assertThat(completed[0], nullValue()); - assertThat(failure[0], sameInstance(error)); - Mockito.verify(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); - Mockito.verifyNoMoreInteractions(client); - } - - public void testUnFollowNotAFollowerIndex() { - IndexMetaData indexMetadata = IndexMetaData.builder("follower-index") - .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - Client client = Mockito.mock(Client.class); - UnfollowFollowIndexStep step = new UnfollowFollowIndexStep(randomStepKey(), randomStepKey(), client); - - Boolean[] completed = new Boolean[1]; - Exception[] failure = new Exception[1]; - step.performAction(indexMetadata, null, new AsyncActionStep.Listener() { - @Override - public void onResponse(boolean complete) { - completed[0] = complete; - } - - @Override - public void onFailure(Exception e) { - failure[0] = e; - } - }); - assertThat(completed[0], is(true)); - assertThat(failure[0], nullValue()); - Mockito.verifyZeroInteractions(client); - } - - private void mockOpenIndexApiCall(IndicesAdminClient indicesClient) { - Mockito.doAnswer(invocation -> { - OpenIndexRequest closeIndexRequest = (OpenIndexRequest) invocation.getArguments()[0]; - assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onResponse(new OpenIndexResponse(true, true)); - return null; - }).when(indicesClient).open(Mockito.any(), Mockito.any()); - } - - private void mockUnfollowApiCall(Client client) { - Mockito.doAnswer(invocation -> { - UnfollowAction.Request request = (UnfollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowerIndex(), equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); - } - - private void mockCloseIndexApiCall(IndicesAdminClient indicesClient) { - Mockito.doAnswer(invocation -> { - CloseIndexRequest closeIndexRequest = (CloseIndexRequest) invocation.getArguments()[0]; - assertThat(closeIndexRequest.indices()[0], equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(indicesClient).close(Mockito.any(), Mockito.any()); - } - - private void mockPauseApiCall(Client client) { - Mockito.doAnswer(invocation -> { - PauseFollowAction.Request request = (PauseFollowAction.Request) invocation.getArguments()[1]; - assertThat(request.getFollowIndex(), equalTo("follower-index")); - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new AcknowledgedResponse(true)); - return null; - }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); - } } From 2b60e546ddd8d5af08c487332cb74671324ea85c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 9 Jan 2019 15:44:04 -0700 Subject: [PATCH 20/35] Remove unused imports --- .../xpack/core/indexlifecycle/CloseFollowerIndexStep.java | 1 - .../xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStep.java index 3c6c3f4c04505..3fb6e145236bc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStep.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.client.Client; -import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; final class CloseFollowerIndexStep extends AbstractUnfollowIndexStep { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java index fd0660ec224c4..528021189e107 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CloseFollowerIndexStepTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; import org.mockito.Mockito; import java.util.Collections; From 533bc4f2a60abe924f9c04abe21f5209dbf7446c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 10 Jan 2019 16:08:50 -0700 Subject: [PATCH 21/35] Allow UnfollowAction in Warm and Cold phases --- .../xpack/core/indexlifecycle/TimeseriesLifecycleType.java | 4 ++-- .../core/indexlifecycle/TimeseriesLifecycleTypeTests.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java index c5c622b814b94..475d6827bf358 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java @@ -35,9 +35,9 @@ public class TimeseriesLifecycleType implements LifecycleType { public static final String TYPE = "timeseries"; static final List VALID_PHASES = Arrays.asList("hot", "warm", "cold", "delete"); static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(UnfollowAction.NAME, RolloverAction.NAME); - static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(ReadOnlyAction.NAME, AllocateAction.NAME, + static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); - static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(AllocateAction.NAME, FreezeAction.NAME); + static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(UnfollowAction.NAME, AllocateAction.NAME, FreezeAction.NAME); static final List ORDERED_VALID_DELETE_ACTIONS = Arrays.asList(DeleteAction.NAME); static final Set VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS); static final Set VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java index a5ec48994aea8..37ae378f3a70a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java @@ -125,7 +125,7 @@ public void testValidateDeletePhase() { Map actions = VALID_DELETE_ACTIONS .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity())); if (randomBoolean()) { - invalidAction = getTestAction(randomFrom("allocate", "rollover", "forcemerge", "shrink")); + invalidAction = getTestAction(randomFrom("allocate", "rollover", "forcemerge", "shrink", "unfollow")); actions.put(invalidAction.getWriteableName(), invalidAction); } Map deletePhase = Collections.singletonMap("delete", From 731d8d2cfbfcc8e8df2e0d7819953e6620e1ca90 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 10 Jan 2019 17:16:45 -0700 Subject: [PATCH 22/35] Add Unfollow action to HLRC --- .../IndexLifecycleNamedXContentProvider.java | 5 +- .../indexlifecycle/LifecyclePolicy.java | 7 +- .../client/indexlifecycle/UnfollowAction.java | 74 +++++++++++++++++++ .../client/IndexLifecycleIT.java | 8 +- .../client/RestHighLevelClientTests.java | 6 +- .../GetLifecyclePolicyResponseTests.java | 3 +- .../LifecyclePolicyMetadataTests.java | 3 +- .../indexlifecycle/LifecyclePolicyTests.java | 13 +++- .../indexlifecycle/UnfollowActionTests.java | 43 +++++++++++ 9 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleNamedXContentProvider.java index 1c22f1e0654f8..db8f261b93e37 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleNamedXContentProvider.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleNamedXContentProvider.java @@ -53,7 +53,10 @@ public List getNamedXContentParsers() { ShrinkAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), - FreezeAction::parse) + FreezeAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, + new ParseField(UnfollowAction.NAME), + UnfollowAction::parse) ); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java index 21a052500a4ae..d8413d55ba42e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java @@ -57,9 +57,10 @@ public class LifecyclePolicy implements ToXContentObject { throw new IllegalArgumentException("ordered " + PHASES_FIELD.getPreferredName() + " are not supported"); }, PHASES_FIELD); - ALLOWED_ACTIONS.put("hot", Sets.newHashSet(RolloverAction.NAME)); - ALLOWED_ACTIONS.put("warm", Sets.newHashSet(AllocateAction.NAME, ForceMergeAction.NAME, ReadOnlyAction.NAME, ShrinkAction.NAME)); - ALLOWED_ACTIONS.put("cold", Sets.newHashSet(AllocateAction.NAME, FreezeAction.NAME)); + ALLOWED_ACTIONS.put("hot", Sets.newHashSet(UnfollowAction.NAME, RolloverAction.NAME)); + ALLOWED_ACTIONS.put("warm", Sets.newHashSet(UnfollowAction.NAME, AllocateAction.NAME, ForceMergeAction.NAME, ReadOnlyAction.NAME, + ShrinkAction.NAME)); + ALLOWED_ACTIONS.put("cold", Sets.newHashSet(UnfollowAction.NAME, AllocateAction.NAME, FreezeAction.NAME)); ALLOWED_ACTIONS.put("delete", Sets.newHashSet(DeleteAction.NAME)); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java new file mode 100644 index 0000000000000..ba25cf937ec8f --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indexlifecycle; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class UnfollowAction implements LifecycleAction, ToXContentObject { + public static final String NAME = "unfollow"; + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, UnfollowAction::new); + + public UnfollowAction() {} + + @Override + public String getName() { + return NAME; + } + + public static UnfollowAction parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return 36970; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + return true; + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java index 08ec5a5b3fe09..4ad6d2e6ce604 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java @@ -48,6 +48,7 @@ import org.elasticsearch.client.indexlifecycle.ShrinkAction; import org.elasticsearch.client.indexlifecycle.StartILMRequest; import org.elasticsearch.client.indexlifecycle.StopILMRequest; +import org.elasticsearch.client.indexlifecycle.UnfollowAction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.hamcrest.Matchers; @@ -144,19 +145,20 @@ public void testStartStopILM() throws Exception { public void testExplainLifecycle() throws Exception { Map lifecyclePhases = new HashMap<>(); - Map hotActions = Collections.singletonMap( - RolloverAction.NAME, - new RolloverAction(null, TimeValue.timeValueHours(50 * 24), null)); + Map hotActions = new HashMap<>(); + hotActions.put(RolloverAction.NAME, new RolloverAction(null, TimeValue.timeValueHours(50 * 24), null)); Phase hotPhase = new Phase("hot", randomFrom(TimeValue.ZERO, null), hotActions); lifecyclePhases.put("hot", hotPhase); Map warmActions = new HashMap<>(); + warmActions.put(UnfollowAction.NAME, new UnfollowAction()); warmActions.put(AllocateAction.NAME, new AllocateAction(null, null, null, Collections.singletonMap("_name", "node-1"))); warmActions.put(ShrinkAction.NAME, new ShrinkAction(1)); warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1000)); lifecyclePhases.put("warm", new Phase("warm", TimeValue.timeValueSeconds(1000), warmActions)); Map coldActions = new HashMap<>(); + coldActions.put(UnfollowAction.NAME, new UnfollowAction()); coldActions.put(AllocateAction.NAME, new AllocateAction(0, null, null, null)); lifecyclePhases.put("cold", new Phase("cold", TimeValue.timeValueSeconds(2000), coldActions)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index a94ab4541f0f9..1d5f0bee665b6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -55,6 +55,7 @@ import org.elasticsearch.client.indexlifecycle.ReadOnlyAction; import org.elasticsearch.client.indexlifecycle.RolloverAction; import org.elasticsearch.client.indexlifecycle.ShrinkAction; +import org.elasticsearch.client.indexlifecycle.UnfollowAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.bytes.BytesReference; @@ -644,7 +645,7 @@ public void testDefaultNamedXContents() { public void testProvidedNamedXContents() { List namedXContents = RestHighLevelClient.getProvidedNamedXContents(); - assertEquals(18, namedXContents.size()); + assertEquals(19, namedXContents.size()); Map, Integer> categories = new HashMap<>(); List names = new ArrayList<>(); for (NamedXContentRegistry.Entry namedXContent : namedXContents) { @@ -668,7 +669,8 @@ public void testProvidedNamedXContents() { assertTrue(names.contains(MeanReciprocalRank.NAME)); assertTrue(names.contains(DiscountedCumulativeGain.NAME)); assertTrue(names.contains(ExpectedReciprocalRank.NAME)); - assertEquals(Integer.valueOf(7), categories.get(LifecycleAction.class)); + assertEquals(Integer.valueOf(8), categories.get(LifecycleAction.class)); + assertTrue(names.contains(UnfollowAction.NAME)); assertTrue(names.contains(AllocateAction.NAME)); assertTrue(names.contains(DeleteAction.NAME)); assertTrue(names.contains(ForceMergeAction.NAME)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java index d703d90d95ed9..6a9f59487d13d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java @@ -67,7 +67,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), FreezeAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), FreezeAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) )); return new NamedXContentRegistry(entries); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java index 93fb69c2ab47d..57dfe220ed63c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java @@ -63,7 +63,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), FreezeAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), FreezeAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) )); return new NamedXContentRegistry(entries); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java index 97c98919d8a88..8cc2a6f8404ea 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java @@ -39,10 +39,10 @@ import static org.hamcrest.Matchers.equalTo; public class LifecyclePolicyTests extends AbstractXContentTestCase { - private static final Set VALID_HOT_ACTIONS = Sets.newHashSet(RolloverAction.NAME); - private static final Set VALID_WARM_ACTIONS = Sets.newHashSet(AllocateAction.NAME, ForceMergeAction.NAME, + private static final Set VALID_HOT_ACTIONS = Sets.newHashSet(UnfollowAction.NAME, RolloverAction.NAME); + private static final Set VALID_WARM_ACTIONS = Sets.newHashSet(UnfollowAction.NAME, AllocateAction.NAME, ForceMergeAction.NAME, ReadOnlyAction.NAME, ShrinkAction.NAME); - private static final Set VALID_COLD_ACTIONS = Sets.newHashSet(AllocateAction.NAME, FreezeAction.NAME); + private static final Set VALID_COLD_ACTIONS = Sets.newHashSet(UnfollowAction.NAME, AllocateAction.NAME, FreezeAction.NAME); private static final Set VALID_DELETE_ACTIONS = Sets.newHashSet(DeleteAction.NAME); private String lifecycleName; @@ -67,7 +67,8 @@ protected NamedXContentRegistry xContentRegistry() { new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), - new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), FreezeAction::parse) + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(FreezeAction.NAME), FreezeAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(UnfollowAction.NAME), UnfollowAction::parse) )); return new NamedXContentRegistry(entries); } @@ -210,6 +211,8 @@ public static LifecyclePolicy createRandomPolicy(String lifecycleName) { return ShrinkActionTests.randomInstance(); case FreezeAction.NAME: return new FreezeAction(); + case UnfollowAction.NAME: + return new UnfollowAction(); default: throw new IllegalArgumentException("invalid action [" + action + "]"); }}; @@ -241,6 +244,8 @@ private LifecycleAction getTestAction(String actionName) { return ShrinkActionTests.randomInstance(); case FreezeAction.NAME: return new FreezeAction(); + case UnfollowAction.NAME: + return new UnfollowAction(); default: throw new IllegalArgumentException("unsupported phase action [" + actionName + "]"); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java new file mode 100644 index 0000000000000..4dd73c5a08ec2 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indexlifecycle; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class UnfollowActionTests extends AbstractXContentTestCase { + + @Override + protected UnfollowAction createTestInstance() { + return new UnfollowAction(); + } + + @Override + protected UnfollowAction doParseInstance(XContentParser parser) throws IOException { + return UnfollowAction.parse(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} From 67f970e65fd73d77b5096acc7bca50037776294a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 11 Jan 2019 12:57:13 +0100 Subject: [PATCH 23/35] added docs for the unfollow action --- .../reference/ilm/policy-definitions.asciidoc | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/reference/ilm/policy-definitions.asciidoc b/docs/reference/ilm/policy-definitions.asciidoc index d00782472b3f4..dd383d402ce7a 100644 --- a/docs/reference/ilm/policy-definitions.asciidoc +++ b/docs/reference/ilm/policy-definitions.asciidoc @@ -579,6 +579,43 @@ PUT _ilm/policy/my_policy -------------------------------------------------- // CONSOLE +[[ilm-unfullow-action]] +==== Unfollow + +This action turns a ccr follower index into a regular index. +This can be desired when moving follower indices into the +next phase. Also certain actions like shrink and rollover +can then be performed safely on follower indices. + +If the unfollow action encounters a follower index then +the following operations will be performed on it: + +* Pauses indexing following for the follower index. +* Closes the follower index. +* Unfollows the follower index. +* Opens the follower index (which is at this point is a regular index). + +The unfollow action does not have any options and +if it encounters a non follower index, then the +unfollow action leaves that index untouched and +lets the next action operate on this index. + +[source,js] +-------------------------------------------------- +PUT _ilm/policy/my_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "unfollow" : {} + } + } + } + } +} +-------------------------------------------------- +// CONSOLE === Full Policy From 272b8d66fb4c3e8ffec0e3902de78a666224114f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 11 Jan 2019 13:43:56 +0100 Subject: [PATCH 24/35] added wait for yellow step. That will run after open index step, so that after unfollow is done then the unfollowed index is ready to handle requests. --- .../core/indexlifecycle/UnfollowAction.java | 9 +- .../indexlifecycle/WaitForYellowStep.java | 80 ++++++++++ .../indexlifecycle/UnfollowActionTests.java | 9 +- .../WaitForYellowStepTests.java | 141 ++++++++++++++++++ 4 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java index b70d5e8441ef8..efa514945e20e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java @@ -42,14 +42,16 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey closeFollowerIndex = new StepKey(phase, NAME, CloseFollowerIndexStep.NAME); StepKey unfollowFollowerIndex = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); StepKey openFollowerIndex = new StepKey(phase, NAME, OpenFollowerIndexStep.NAME); + StepKey waitForYellowStep = new StepKey(phase, NAME, WaitForYellowStep.NAME); WaitForIndexingCompleteStep step1 = new WaitForIndexingCompleteStep(indexingComplete, waitForFollowShardTasks); WaitForFollowShardTasksStep step2 = new WaitForFollowShardTasksStep(waitForFollowShardTasks, pauseFollowerIndex, client); PauseFollowerIndexStep step3 = new PauseFollowerIndexStep(pauseFollowerIndex, closeFollowerIndex, client); CloseFollowerIndexStep step4 = new CloseFollowerIndexStep(closeFollowerIndex, unfollowFollowerIndex, client); UnfollowFollowIndexStep step5 = new UnfollowFollowIndexStep(unfollowFollowerIndex, openFollowerIndex, client); - OpenFollowerIndexStep step6 = new OpenFollowerIndexStep(openFollowerIndex, nextStepKey, client); - return Arrays.asList(step1, step2, step3, step4, step5, step6); + OpenFollowerIndexStep step6 = new OpenFollowerIndexStep(openFollowerIndex, waitForYellowStep, client); + WaitForYellowStep step7 = new WaitForYellowStep(waitForYellowStep, nextStepKey, client); + return Arrays.asList(step1, step2, step3, step4, step5, step6, step7); } @Override @@ -60,8 +62,9 @@ public List toStepKeys(String phase) { StepKey closeFollowerIndexStep = new StepKey(phase, NAME, CloseFollowerIndexStep.NAME); StepKey unfollowIndexStep = new StepKey(phase, NAME, UnfollowFollowIndexStep.NAME); StepKey openFollowerIndexStep = new StepKey(phase, NAME, OpenFollowerIndexStep.NAME); + StepKey waitForYellowStep = new StepKey(phase, NAME, WaitForYellowStep.NAME); return Arrays.asList(indexingCompleteStep, waitForFollowShardTasksStep, pauseFollowerIndexStep, - closeFollowerIndexStep, unfollowIndexStep, openFollowerIndexStep); + closeFollowerIndexStep, unfollowIndexStep, openFollowerIndexStep, waitForYellowStep); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java new file mode 100644 index 0000000000000..d0c966c6ccc7c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +class WaitForYellowStep extends AsyncWaitStep { + + static final String NAME = "wait-for-yellow-step"; + + WaitForYellowStep(StepKey key, StepKey nextStepKey, Client client) { + super(key, nextStepKey, client); + } + + @Override + public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) { + String indexName = indexMetaData.getIndex().getName(); + ClusterHealthRequest request = new ClusterHealthRequest(indexName); + request.waitForYellowStatus(); + CheckedConsumer handler = clusterHealthResponse -> { + boolean success = clusterHealthResponse.isTimedOut() == false; + if (success) { + listener.onResponse(success, null); + } else { + listener.onResponse(false, new Info()); + } + }; + getClient().admin().cluster().health(request, ActionListener.wrap(handler, listener::onFailure)); + } + + static final class Info implements ToXContentObject { + + static final ParseField MESSAGE_FIELD = new ParseField("message"); + + private final String message; + + Info() { + this.message = "cluster health request timed out waiting for yellow status"; + } + + String getMessage() { + return message; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(MESSAGE_FIELD.getPreferredName(), message); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Info info = (Info) o; + return Objects.equals(getMessage(), info.getMessage()); + } + + @Override + public int hashCode() { + return Objects.hash(getMessage()); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java index e107b53153f8c..42f299a8aeafd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowActionTests.java @@ -39,7 +39,7 @@ public void testToSteps() { randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); assertThat(steps, notNullValue()); - assertThat(steps.size(), equalTo(6)); + assertThat(steps.size(), equalTo(7)); StepKey expectedFirstStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForIndexingCompleteStep.NAME); StepKey expectedSecondStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForFollowShardTasksStep.NAME); @@ -47,6 +47,7 @@ public void testToSteps() { StepKey expectedFourthStepKey = new StepKey(phase, UnfollowAction.NAME, CloseFollowerIndexStep.NAME); StepKey expectedFifthStepKey = new StepKey(phase, UnfollowAction.NAME, UnfollowFollowIndexStep.NAME); StepKey expectedSixthStepKey = new StepKey(phase, UnfollowAction.NAME, OpenFollowerIndexStep.NAME); + StepKey expectedSeventhStepKey = new StepKey(phase, UnfollowAction.NAME, WaitForYellowStep.NAME); WaitForIndexingCompleteStep firstStep = (WaitForIndexingCompleteStep) steps.get(0); assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey)); @@ -70,6 +71,10 @@ public void testToSteps() { OpenFollowerIndexStep sixthStep = (OpenFollowerIndexStep) steps.get(5); assertThat(sixthStep.getKey(), equalTo(expectedSixthStepKey)); - assertThat(sixthStep.getNextStepKey(), equalTo(nextStepKey)); + assertThat(sixthStep.getNextStepKey(), equalTo(expectedSeventhStepKey)); + + WaitForYellowStep seventhStep = (WaitForYellowStep) steps.get(6); + assertThat(seventhStep.getKey(), equalTo(expectedSeventhStepKey)); + assertThat(seventhStep.getNextStepKey(), equalTo(nextStepKey)); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java new file mode 100644 index 0000000000000..24df5352accfd --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.client.AdminClient; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ClusterAdminClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; +import org.mockito.Mockito; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.IsNull.notNullValue; + +public class WaitForYellowStepTests extends AbstractStepTestCase { + + @Override + protected WaitForYellowStep createRandomInstance() { + StepKey stepKey = randomStepKey(); + StepKey nextStepKey = randomStepKey(); + return new WaitForYellowStep(stepKey, nextStepKey, Mockito.mock(Client.class)); + } + + @Override + protected WaitForYellowStep mutateInstance(WaitForYellowStep instance) { + StepKey key = instance.getKey(); + StepKey nextKey = instance.getNextStepKey(); + + if (randomBoolean()) { + key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } else { + nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } + + return new WaitForYellowStep(key, nextKey, instance.getClient()); + } + + @Override + protected WaitForYellowStep copyInstance(WaitForYellowStep instance) { + return new WaitForYellowStep(instance.getKey(), instance.getNextStepKey(), instance.getClient()); + } + + public void testConditionMet() { + IndexMetaData indexMetadata = IndexMetaData.builder("former-follower-index") + .settings(settings(Version.CURRENT)) + .numberOfShards(2) + .numberOfReplicas(0) + .build(); + Client client = Mockito.mock(Client.class); + String indexName = indexMetadata.getIndex().getName(); + ClusterHealthResponse healthResponse = new ClusterHealthResponse("_cluster", new String[]{indexName}, ClusterState.EMPTY_STATE); + mockClusterHealthCall(client, indexMetadata.getIndex().getName(), healthResponse); + + WaitForYellowStep step = new WaitForYellowStep(randomStepKey(), randomStepKey(), client); + final boolean[] conditionMetHolder = new boolean[1]; + final ToXContentObject[] informationContextHolder = new ToXContentObject[1]; + final Exception[] exceptionHolder = new Exception[1]; + step.evaluateCondition(indexMetadata, new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject informationContext) { + conditionMetHolder[0] = conditionMet; + informationContextHolder[0] = informationContext; + } + + @Override + public void onFailure(Exception e) { + exceptionHolder[0] = e; + } + }); + + assertThat(conditionMetHolder[0], is(true)); + assertThat(informationContextHolder[0], nullValue()); + assertThat(exceptionHolder[0], nullValue()); + } + + public void testConditionNotMet() { + IndexMetaData indexMetadata = IndexMetaData.builder("former-follower-index") + .settings(settings(Version.CURRENT)) + .numberOfShards(2) + .numberOfReplicas(0) + .build(); + Client client = Mockito.mock(Client.class); + String indexName = indexMetadata.getIndex().getName(); + ClusterHealthResponse healthResponse = new ClusterHealthResponse("_cluster", new String[]{indexName}, ClusterState.EMPTY_STATE); + healthResponse.setTimedOut(true); + mockClusterHealthCall(client, indexMetadata.getIndex().getName(), healthResponse); + + WaitForYellowStep step = new WaitForYellowStep(randomStepKey(), randomStepKey(), client); + final boolean[] conditionMetHolder = new boolean[1]; + final ToXContentObject[] informationContextHolder = new ToXContentObject[1]; + final Exception[] exceptionHolder = new Exception[1]; + step.evaluateCondition(indexMetadata, new AsyncWaitStep.Listener() { + @Override + public void onResponse(boolean conditionMet, ToXContentObject informationContext) { + conditionMetHolder[0] = conditionMet; + informationContextHolder[0] = informationContext; + } + + @Override + public void onFailure(Exception e) { + exceptionHolder[0] = e; + } + }); + + assertThat(conditionMetHolder[0], is(false)); + assertThat(informationContextHolder[0], notNullValue()); + assertThat(exceptionHolder[0], nullValue()); + WaitForYellowStep.Info info = (WaitForYellowStep.Info) informationContextHolder[0]; + assertThat(info.getMessage(), equalTo("cluster health request timed out waiting for yellow status")); + } + + private void mockClusterHealthCall(Client client, String expectedIndexName, ClusterHealthResponse healthResponse) { + AdminClient adminClient = Mockito.mock(AdminClient.class); + ClusterAdminClient clusterAdminClient = Mockito.mock(ClusterAdminClient.class); + Mockito.when(client.admin()).thenReturn(adminClient); + Mockito.when(adminClient.cluster()).thenReturn(clusterAdminClient); + Mockito.doAnswer(invocationOnMock -> { + ClusterHealthRequest request = (ClusterHealthRequest) invocationOnMock.getArguments()[0]; + assertThat(request.indices().length, equalTo(1)); + assertThat(request.indices()[0], equalTo(expectedIndexName)); + assertThat(request.waitForStatus(), equalTo(ClusterHealthStatus.YELLOW)); + + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; + listener.onResponse(healthResponse); + return null; + }).when(clusterAdminClient).health(Mockito.any(), Mockito.any()); + } +} From 04164902bbf6937f4d7afde77d1b313593817a56 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 14 Jan 2019 09:42:47 +0100 Subject: [PATCH 25/35] docs iter --- docs/reference/ilm/policy-definitions.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/ilm/policy-definitions.asciidoc b/docs/reference/ilm/policy-definitions.asciidoc index eb88fb3c87f19..0c1a81fba8e6c 100644 --- a/docs/reference/ilm/policy-definitions.asciidoc +++ b/docs/reference/ilm/policy-definitions.asciidoc @@ -571,13 +571,13 @@ PUT _ilm/policy/my_policy -------------------------------------------------- // CONSOLE -[[ilm-unfullow-action]] +[[ilm-unfollow-action]] ==== Unfollow -This action turns a ccr follower index into a regular index. -This can be desired when moving follower indices into the -next phase. Also certain actions like shrink and rollover -can then be performed safely on follower indices. +This action turns a {ref}/ccr-apis.html[ccr] follower index +into a regular index. This can be desired when moving follower +indices into the next phase. Also certain actions like shrink +and rollover can then be performed safely on follower indices. If the unfollow action encounters a follower index then the following operations will be performed on it: From 443fc4ac41b8d18cb45ccf5650616a8672320b96 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 14 Jan 2019 10:19:28 +0100 Subject: [PATCH 26/35] changed WaitForYellowStep to be a AsyncWaitStep --- .../core/indexlifecycle/UnfollowAction.java | 2 +- .../indexlifecycle/WaitForYellowStep.java | 36 ++--- .../WaitForYellowStepTests.java | 127 ++++++------------ 3 files changed, 57 insertions(+), 108 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java index efa514945e20e..20a0fb75b9daa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UnfollowAction.java @@ -50,7 +50,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { CloseFollowerIndexStep step4 = new CloseFollowerIndexStep(closeFollowerIndex, unfollowFollowerIndex, client); UnfollowFollowIndexStep step5 = new UnfollowFollowIndexStep(unfollowFollowerIndex, openFollowerIndex, client); OpenFollowerIndexStep step6 = new OpenFollowerIndexStep(openFollowerIndex, waitForYellowStep, client); - WaitForYellowStep step7 = new WaitForYellowStep(waitForYellowStep, nextStepKey, client); + WaitForYellowStep step7 = new WaitForYellowStep(waitForYellowStep, nextStepKey); return Arrays.asList(step1, step2, step3, step4, step5, step6, step7); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java index d0c966c6ccc7c..73e4f2ebd49b8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java @@ -5,41 +5,31 @@ */ package org.elasticsearch.xpack.core.indexlifecycle; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.Index; import java.io.IOException; import java.util.Objects; -class WaitForYellowStep extends AsyncWaitStep { +class WaitForYellowStep extends ClusterStateWaitStep { static final String NAME = "wait-for-yellow-step"; - WaitForYellowStep(StepKey key, StepKey nextStepKey, Client client) { - super(key, nextStepKey, client); + WaitForYellowStep(StepKey key, StepKey nextStepKey) { + super(key, nextStepKey); } @Override - public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) { - String indexName = indexMetaData.getIndex().getName(); - ClusterHealthRequest request = new ClusterHealthRequest(indexName); - request.waitForYellowStatus(); - CheckedConsumer handler = clusterHealthResponse -> { - boolean success = clusterHealthResponse.isTimedOut() == false; - if (success) { - listener.onResponse(success, null); - } else { - listener.onResponse(false, new Info()); - } - }; - getClient().admin().cluster().health(request, ActionListener.wrap(handler, listener::onFailure)); + public Result isConditionMet(Index index, ClusterState clusterState) { + boolean indexIsAtLeastYellow = clusterState.routingTable().index(index).allPrimaryShardsActive(); + if (indexIsAtLeastYellow) { + return new Result(true, null); + } else { + return new Result(false, new Info()); + } } static final class Info implements ToXContentObject { @@ -49,7 +39,7 @@ static final class Info implements ToXContentObject { private final String message; Info() { - this.message = "cluster health request timed out waiting for yellow status"; + this.message = "index is red; not all primary shards are active"; } String getMessage() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java index 24df5352accfd..9ffd72103f4fd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java @@ -6,18 +6,16 @@ package org.elasticsearch.xpack.core.indexlifecycle; import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.AdminClient; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.ClusterAdminClient; +import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; -import org.mockito.Mockito; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -30,7 +28,7 @@ public class WaitForYellowStepTests extends AbstractStepTestCase { - ClusterHealthRequest request = (ClusterHealthRequest) invocationOnMock.getArguments()[0]; - assertThat(request.indices().length, equalTo(1)); - assertThat(request.indices()[0], equalTo(expectedIndexName)); - assertThat(request.waitForStatus(), equalTo(ClusterHealthStatus.YELLOW)); - - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(healthResponse); - return null; - }).when(clusterAdminClient).health(Mockito.any(), Mockito.any()); + ShardRouting shardRouting = + TestShardRouting.newShardRouting("index2", 0, "1", true, ShardRoutingState.INITIALIZING); + IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex()) + .addShard(shardRouting).build(); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().put(indexMetadata, true).build()) + .routingTable(RoutingTable.builder().add(indexRoutingTable).build()) + .build(); + + WaitForYellowStep step = new WaitForYellowStep(randomStepKey(), randomStepKey()); + ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); + assertThat(result.isComplete(), is(false)); + WaitForYellowStep.Info info = (WaitForYellowStep.Info) result.getInfomationContext(); + assertThat(info, notNullValue()); + assertThat(info.getMessage(), equalTo("index is red; not all primary shards are active")); } } From 5abf5578a027cfce17edc22401e4d8cf7d080919 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 14 Jan 2019 16:20:28 +0100 Subject: [PATCH 27/35] take into account no IndexRoutingTable --- .../indexlifecycle/WaitForYellowStep.java | 16 +++++++++++---- .../WaitForYellowStepTests.java | 20 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java index 73e4f2ebd49b8..75be80199e9b2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStep.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.core.indexlifecycle; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -24,11 +26,17 @@ class WaitForYellowStep extends ClusterStateWaitStep { @Override public Result isConditionMet(Index index, ClusterState clusterState) { - boolean indexIsAtLeastYellow = clusterState.routingTable().index(index).allPrimaryShardsActive(); + RoutingTable routingTable = clusterState.routingTable(); + IndexRoutingTable indexShardRoutingTable = routingTable.index(index); + if (indexShardRoutingTable == null) { + return new Result(false, new Info("index is red; no IndexRoutingTable")); + } + + boolean indexIsAtLeastYellow = indexShardRoutingTable.allPrimaryShardsActive(); if (indexIsAtLeastYellow) { return new Result(true, null); } else { - return new Result(false, new Info()); + return new Result(false, new Info("index is red; not all primary shards are active")); } } @@ -38,8 +46,8 @@ static final class Info implements ToXContentObject { private final String message; - Info() { - this.message = "index is red; not all primary shards are active"; + Info(String message) { + this.message = message; } String getMessage() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java index 9ffd72103f4fd..6c3915d87cde4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForYellowStepTests.java @@ -97,4 +97,24 @@ public void testConditionNotMet() { assertThat(info, notNullValue()); assertThat(info.getMessage(), equalTo("index is red; not all primary shards are active")); } + + public void testConditionNotMetNoIndexRoutingTable() { + IndexMetaData indexMetadata = IndexMetaData.builder("former-follower-index") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().put(indexMetadata, true).build()) + .routingTable(RoutingTable.builder().build()) + .build(); + + WaitForYellowStep step = new WaitForYellowStep(randomStepKey(), randomStepKey()); + ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); + assertThat(result.isComplete(), is(false)); + WaitForYellowStep.Info info = (WaitForYellowStep.Info) result.getInfomationContext(); + assertThat(info, notNullValue()); + assertThat(info.getMessage(), equalTo("index is red; no IndexRoutingTable")); + } } From caa13ce5d6a697fe51d3c885f6edf05867009d08 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 14 Jan 2019 15:20:30 -0700 Subject: [PATCH 28/35] Add Unfollow to Warm/Cold phase ordering test --- .../TimeseriesLifecycleTypeTests.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java index 37ae378f3a70a..face415ec1b9c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java @@ -316,6 +316,15 @@ public void testGetNextActionName() { assertInvalidAction("hot", ShrinkAction.NAME, new String[] { RolloverAction.NAME }); // Warm Phase + assertNextActionName("warm", UnfollowAction.NAME, ReadOnlyAction.NAME, + new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, AllocateAction.NAME, + new String[] { AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, ShrinkAction.NAME, + new String[] { ShrinkAction.NAME, ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, ForceMergeAction.NAME, new String[] { ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, null, new String[] {}); + assertNextActionName("warm", ReadOnlyAction.NAME, AllocateAction.NAME, new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); assertNextActionName("warm", ReadOnlyAction.NAME, ShrinkAction.NAME, @@ -360,6 +369,10 @@ public void testGetNextActionName() { new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); // Cold Phase + assertNextActionName("cold", UnfollowAction.NAME, AllocateAction.NAME, + new String[] {AllocateAction.NAME, FreezeAction.NAME}); + assertNextActionName("cold", UnfollowAction.NAME, FreezeAction.NAME, new String[] {FreezeAction.NAME}); + assertNextActionName("cold", UnfollowAction.NAME, null, new String[] {}); assertNextActionName("cold", AllocateAction.NAME, null, new String[] { AllocateAction.NAME }); assertNextActionName("cold", AllocateAction.NAME, null, new String[] {}); assertNextActionName("cold", AllocateAction.NAME, null, new String[] {}); @@ -383,6 +396,7 @@ public void testGetNextActionName() { assertInvalidAction("delete", ReadOnlyAction.NAME, new String[] { DeleteAction.NAME }); assertInvalidAction("delete", RolloverAction.NAME, new String[] { DeleteAction.NAME }); assertInvalidAction("delete", ShrinkAction.NAME, new String[] { DeleteAction.NAME }); + assertInvalidAction("delete", UnfollowAction.NAME, new String[] { DeleteAction.NAME }); Phase phase = new Phase("foo", TimeValue.ZERO, Collections.emptyMap()); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, From 5d3d661b98fe96ebccf90ab486c4d58941d12e4d Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 15 Jan 2019 11:00:53 +0100 Subject: [PATCH 29/35] OpenFollowerIndexStep cannot extend from AbstractUnfollowIndexStep, because after the unfollow step it is no longer a follower index. --- .../indexlifecycle/OpenFollowerIndexStep.java | 8 +++--- .../OpenFollowerIndexStepTests.java | 27 ++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStep.java index a95fa1d97cc9f..7ba2c4633ab99 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStep.java @@ -8,8 +8,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; -final class OpenFollowerIndexStep extends AbstractUnfollowIndexStep { +final class OpenFollowerIndexStep extends AsyncActionStep { static final String NAME = "open-follower-index"; @@ -18,8 +20,8 @@ final class OpenFollowerIndexStep extends AbstractUnfollowIndexStep { } @Override - void innerPerformAction(String followerIndex, Listener listener) { - OpenIndexRequest request = new OpenIndexRequest(followerIndex); + public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener) { + OpenIndexRequest request = new OpenIndexRequest(indexMetaData.getIndex().getName()); getClient().admin().indices().open(request, ActionListener.wrap( r -> { assert r.isAcknowledged() : "open index response is not acknowledged"; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStepTests.java index bd254fe0245ff..2d5086ec88fac 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OpenFollowerIndexStepTests.java @@ -23,11 +23,32 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -public class OpenFollowerIndexStepTests extends AbstractUnfollowIndexStepTestCase { +public class OpenFollowerIndexStepTests extends AbstractStepTestCase { @Override - protected OpenFollowerIndexStep newInstance(Step.StepKey key, Step.StepKey nextKey, Client client) { - return new OpenFollowerIndexStep(key, nextKey, client); + protected OpenFollowerIndexStep createRandomInstance() { + Step.StepKey stepKey = randomStepKey(); + Step.StepKey nextStepKey = randomStepKey(); + return new OpenFollowerIndexStep(stepKey, nextStepKey, Mockito.mock(Client.class)); + } + + @Override + protected OpenFollowerIndexStep mutateInstance(OpenFollowerIndexStep instance) { + Step.StepKey key = instance.getKey(); + Step.StepKey nextKey = instance.getNextStepKey(); + + if (randomBoolean()) { + key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } else { + nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + } + + return new OpenFollowerIndexStep(key, nextKey, instance.getClient()); + } + + @Override + protected OpenFollowerIndexStep copyInstance(OpenFollowerIndexStep instance) { + return new OpenFollowerIndexStep(instance.getKey(), instance.getNextStepKey(), instance.getClient()); } public void testOpenFollowingIndex() { From f7e74d4e09348f3e18d0af1cac807c81980515d4 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 15 Jan 2019 12:27:22 -0700 Subject: [PATCH 30/35] Add `indexing_complete` setting to Wait step info --- .../indexlifecycle/WaitForIndexingCompleteStep.java | 12 ++++++++---- .../WaitForIndexingCompleteStepTests.java | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java index db3ede990ab9c..f013838aaa2cd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java @@ -46,19 +46,22 @@ public Result isConditionMet(Index index, ClusterState clusterState) { if (indexingComplete) { return new Result(true, null); } else { - return new Result(false, new Info()); + return new Result(false, new Info(indexingComplete)); } } static final class Info implements ToXContentObject { static final ParseField MESSAGE_FIELD = new ParseField("message"); + static final ParseField INDEXING_COMPLETE = new ParseField(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE); private final String message; + private final boolean indexingComplete; - Info() { - this.message = "the [" + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE + - "] setting has not been set to true on the leader index"; + Info(boolean indexingComplete) { + this.indexingComplete = indexingComplete; + this.message = "waiting for the [" + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE + + "] setting to be set to true on the leader index, it is currently [" + indexingComplete + "]"; } String getMessage() { @@ -69,6 +72,7 @@ String getMessage() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(MESSAGE_FIELD.getPreferredName(), message); + builder.field(INDEXING_COMPLETE.getPreferredName(), indexingComplete); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java index 4984e33eee0ab..c31a91ee1c735 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java @@ -106,8 +106,8 @@ public void testConditionNotMet() { assertThat(result.isComplete(), is(false)); assertThat(result.getInfomationContext(), notNullValue()); WaitForIndexingCompleteStep.Info info = (WaitForIndexingCompleteStep.Info) result.getInfomationContext(); - assertThat(info.getMessage(), equalTo("the [index.lifecycle.indexing_complete] setting has not been set to " + - "true on the leader index")); + assertThat(info.getMessage(), equalTo("waiting for the [index.lifecycle.indexing_complete] setting to be set to " + + "true on the leader index, it is currently [false]")); } public void testIndexDeleted() { From 7c1749b7f6856bb21dd695b9dc9934706256dcbd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 15 Jan 2019 15:38:18 -0700 Subject: [PATCH 31/35] Added Unfollow to the Actions docs ToC --- docs/reference/ilm/policy-definitions.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/reference/ilm/policy-definitions.asciidoc b/docs/reference/ilm/policy-definitions.asciidoc index 9dfd963cb6d6d..ade5b0b74d79e 100644 --- a/docs/reference/ilm/policy-definitions.asciidoc +++ b/docs/reference/ilm/policy-definitions.asciidoc @@ -86,14 +86,17 @@ The below list shows the actions which are available in each phase. * Hot - <> + - <> * Warm - <> - <> - <> - <> + - <> * Cold - <> - <> + - <> * Delete - <> From fcf34e61f0dbf13ef2e4de8b059b692276762b09 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 15 Jan 2019 16:15:14 -0700 Subject: [PATCH 32/35] Switch Info to hardcoded false --- .../WaitForIndexingCompleteStep.java | 14 ++++++-------- .../WaitForIndexingCompleteStepTests.java | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java index f013838aaa2cd..3f795a88dd85b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStep.java @@ -46,22 +46,20 @@ public Result isConditionMet(Index index, ClusterState clusterState) { if (indexingComplete) { return new Result(true, null); } else { - return new Result(false, new Info(indexingComplete)); + return new Result(false, new IndexingNotCompleteInfo()); } } - static final class Info implements ToXContentObject { + static final class IndexingNotCompleteInfo implements ToXContentObject { static final ParseField MESSAGE_FIELD = new ParseField("message"); static final ParseField INDEXING_COMPLETE = new ParseField(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE); private final String message; - private final boolean indexingComplete; - Info(boolean indexingComplete) { - this.indexingComplete = indexingComplete; + IndexingNotCompleteInfo() { this.message = "waiting for the [" + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE + - "] setting to be set to true on the leader index, it is currently [" + indexingComplete + "]"; + "] setting to be set to true on the leader index, it is currently [false]"; } String getMessage() { @@ -72,7 +70,7 @@ String getMessage() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(MESSAGE_FIELD.getPreferredName(), message); - builder.field(INDEXING_COMPLETE.getPreferredName(), indexingComplete); + builder.field(INDEXING_COMPLETE.getPreferredName(), false); builder.endObject(); return builder; } @@ -81,7 +79,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Info info = (Info) o; + IndexingNotCompleteInfo info = (IndexingNotCompleteInfo) o; return Objects.equals(getMessage(), info.getMessage()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java index c31a91ee1c735..83683e477cd1c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java @@ -105,7 +105,7 @@ public void testConditionNotMet() { ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); assertThat(result.isComplete(), is(false)); assertThat(result.getInfomationContext(), notNullValue()); - WaitForIndexingCompleteStep.Info info = (WaitForIndexingCompleteStep.Info) result.getInfomationContext(); + WaitForIndexingCompleteStep.IndexingNotCompleteInfo info = (WaitForIndexingCompleteStep.IndexingNotCompleteInfo) result.getInfomationContext(); assertThat(info.getMessage(), equalTo("waiting for the [index.lifecycle.indexing_complete] setting to be set to " + "true on the leader index, it is currently [false]")); } From c02f8cb7fb9a5d680a8af1882d6c7658bc869b4e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 16 Jan 2019 09:31:08 +0100 Subject: [PATCH 33/35] fixed checkstyle violation --- .../core/indexlifecycle/WaitForIndexingCompleteStepTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java index 83683e477cd1c..41a9c5983a78c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForIndexingCompleteStepTests.java @@ -105,7 +105,8 @@ public void testConditionNotMet() { ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); assertThat(result.isComplete(), is(false)); assertThat(result.getInfomationContext(), notNullValue()); - WaitForIndexingCompleteStep.IndexingNotCompleteInfo info = (WaitForIndexingCompleteStep.IndexingNotCompleteInfo) result.getInfomationContext(); + WaitForIndexingCompleteStep.IndexingNotCompleteInfo info = + (WaitForIndexingCompleteStep.IndexingNotCompleteInfo) result.getInfomationContext(); assertThat(info.getMessage(), equalTo("waiting for the [index.lifecycle.indexing_complete] setting to be set to " + "true on the leader index, it is currently [false]")); } From 3ee62986e58babdc304ba88f3ef3aabcef572bc8 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 17 Jan 2019 12:11:02 -0700 Subject: [PATCH 34/35] Fix a couple tests --- .../core/indexlifecycle/TimeseriesLifecycleTypeTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java index 193ea92485f4e..7d46d9162ff88 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java @@ -165,7 +165,7 @@ public void testGetOrderedActionsHot() { Phase hotPhase = new Phase("hot", TimeValue.ZERO, actions); List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(hotPhase); assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_HOT_ACTIONS)); - assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); + assertThat(orderedActions.indexOf(TEST_UNFOLLOW_ACTION), equalTo(0)); } public void testGetOrderedActionsWarm() { @@ -174,7 +174,7 @@ public void testGetOrderedActionsWarm() { Phase warmPhase = new Phase("warm", TimeValue.ZERO, actions); List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(warmPhase); assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_WARM_ACTIONS)); - assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); + assertThat(orderedActions.indexOf(TEST_UNFOLLOW_ACTION), equalTo(0)); } public void testGetOrderedActionsCold() { @@ -183,7 +183,7 @@ public void testGetOrderedActionsCold() { Phase coldPhase = new Phase("cold", TimeValue.ZERO, actions); List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(coldPhase); assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_COLD_ACTIONS)); - assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); + assertThat(orderedActions.indexOf(TEST_UNFOLLOW_ACTION), equalTo(0)); } public void testGetOrderedActionsDelete() { From f1c05bd2cbcf4fedc502682245f546d5d201b1d7 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 17 Jan 2019 17:42:41 -0700 Subject: [PATCH 35/35] Run SetPriority before Unfollow This is necessary, at least in the Hot phase, so that the priority will be set before waiting for the index to unfollow, which is much more likely what the user expects. --- .../TimeseriesLifecycleType.java | 6 +- .../TimeseriesLifecycleTypeTests.java | 62 +++++++++++-------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java index e96626bfe490b..4d1c770cea4bc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java @@ -34,10 +34,10 @@ public class TimeseriesLifecycleType implements LifecycleType { public static final String TYPE = "timeseries"; static final List VALID_PHASES = Arrays.asList("hot", "warm", "cold", "delete"); - static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(UnfollowAction.NAME, SetPriorityAction.NAME, RolloverAction.NAME); - static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(UnfollowAction.NAME, SetPriorityAction.NAME, ReadOnlyAction.NAME, + static final List ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME); + static final List ORDERED_VALID_WARM_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME); - static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(UnfollowAction.NAME, SetPriorityAction.NAME, AllocateAction.NAME, + static final List ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME, FreezeAction.NAME); static final List ORDERED_VALID_DELETE_ACTIONS = Arrays.asList(DeleteAction.NAME); static final Set VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java index 7d46d9162ff88..4efb34873d471 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java @@ -165,7 +165,7 @@ public void testGetOrderedActionsHot() { Phase hotPhase = new Phase("hot", TimeValue.ZERO, actions); List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(hotPhase); assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_HOT_ACTIONS)); - assertThat(orderedActions.indexOf(TEST_UNFOLLOW_ACTION), equalTo(0)); + assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); } public void testGetOrderedActionsWarm() { @@ -174,7 +174,7 @@ public void testGetOrderedActionsWarm() { Phase warmPhase = new Phase("warm", TimeValue.ZERO, actions); List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(warmPhase); assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_WARM_ACTIONS)); - assertThat(orderedActions.indexOf(TEST_UNFOLLOW_ACTION), equalTo(0)); + assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); } public void testGetOrderedActionsCold() { @@ -183,7 +183,7 @@ public void testGetOrderedActionsCold() { Phase coldPhase = new Phase("cold", TimeValue.ZERO, actions); List orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(coldPhase); assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_COLD_ACTIONS)); - assertThat(orderedActions.indexOf(TEST_UNFOLLOW_ACTION), equalTo(0)); + assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0)); } public void testGetOrderedActionsDelete() { @@ -306,14 +306,14 @@ public void testGetPreviousPhaseName() { public void testGetNextActionName() { // Hot Phase + assertNextActionName("hot", SetPriorityAction.NAME, UnfollowAction.NAME, + new String[] {UnfollowAction.NAME, RolloverAction.NAME}); + assertNextActionName("hot", SetPriorityAction.NAME, RolloverAction.NAME, new String[]{RolloverAction.NAME}); assertNextActionName("hot", SetPriorityAction.NAME, null, new String[] {}); - assertNextActionName("hot", SetPriorityAction.NAME, RolloverAction.NAME, new String[]{SetPriorityAction.NAME, RolloverAction.NAME}); + assertNextActionName("hot", RolloverAction.NAME, null, new String[] {}); assertNextActionName("hot", RolloverAction.NAME, null, new String[] { RolloverAction.NAME }); - assertNextActionName("hot", UnfollowAction.NAME, null, new String[] {}); - assertNextActionName("hot", UnfollowAction.NAME, null, new String[] {UnfollowAction.NAME}); - assertNextActionName("hot", UnfollowAction.NAME, RolloverAction.NAME, - new String[] {UnfollowAction.NAME, RolloverAction.NAME}); + assertInvalidAction("hot", "foo", new String[] { RolloverAction.NAME }); assertInvalidAction("hot", AllocateAction.NAME, new String[] { RolloverAction.NAME }); assertInvalidAction("hot", DeleteAction.NAME, new String[] { RolloverAction.NAME }); @@ -322,17 +322,9 @@ public void testGetNextActionName() { assertInvalidAction("hot", ShrinkAction.NAME, new String[] { RolloverAction.NAME }); // Warm Phase - assertNextActionName("warm", UnfollowAction.NAME, SetPriorityAction.NAME, - new String[] { SetPriorityAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); - assertNextActionName("warm", UnfollowAction.NAME, ReadOnlyAction.NAME, - new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); - assertNextActionName("warm", UnfollowAction.NAME, AllocateAction.NAME, - new String[] { AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); - assertNextActionName("warm", UnfollowAction.NAME, ShrinkAction.NAME, - new String[] { ShrinkAction.NAME, ForceMergeAction.NAME }); - assertNextActionName("warm", UnfollowAction.NAME, ForceMergeAction.NAME, new String[] { ForceMergeAction.NAME }); - assertNextActionName("warm", UnfollowAction.NAME, null, new String[] {}); - + assertNextActionName("warm", SetPriorityAction.NAME, UnfollowAction.NAME, + new String[]{SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, + ShrinkAction.NAME, ForceMergeAction.NAME}); assertNextActionName("warm", SetPriorityAction.NAME, ReadOnlyAction.NAME, new String[]{SetPriorityAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME}); assertNextActionName("warm", SetPriorityAction.NAME, AllocateAction.NAME, @@ -343,6 +335,17 @@ public void testGetNextActionName() { new String[]{SetPriorityAction.NAME, ForceMergeAction.NAME}); assertNextActionName("warm", SetPriorityAction.NAME, null, new String[]{SetPriorityAction.NAME}); + assertNextActionName("warm", UnfollowAction.NAME, ReadOnlyAction.NAME, + new String[] { SetPriorityAction.NAME, ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, ReadOnlyAction.NAME, + new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, AllocateAction.NAME, + new String[] { AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, ShrinkAction.NAME, + new String[] { ShrinkAction.NAME, ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, ForceMergeAction.NAME, new String[] { ForceMergeAction.NAME }); + assertNextActionName("warm", UnfollowAction.NAME, null, new String[] {}); + assertNextActionName("warm", ReadOnlyAction.NAME, AllocateAction.NAME, new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); assertNextActionName("warm", ReadOnlyAction.NAME, ShrinkAction.NAME, @@ -387,21 +390,27 @@ public void testGetNextActionName() { new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME }); // Cold Phase - assertNextActionName("cold", UnfollowAction.NAME, SetPriorityAction.NAME, + assertNextActionName("cold", SetPriorityAction.NAME, UnfollowAction.NAME, + new String[]{UnfollowAction.NAME, SetPriorityAction.NAME, FreezeAction.NAME}); + assertNextActionName("cold", SetPriorityAction.NAME, FreezeAction.NAME, + new String[]{SetPriorityAction.NAME, FreezeAction.NAME}); + assertNextActionName("cold", SetPriorityAction.NAME, AllocateAction.NAME, + new String[]{SetPriorityAction.NAME, AllocateAction.NAME}); + assertNextActionName("cold", SetPriorityAction.NAME, null, new String[] { SetPriorityAction.NAME }); + assertNextActionName("cold", SetPriorityAction.NAME, null, new String[] {}); + + assertNextActionName("cold", UnfollowAction.NAME, AllocateAction.NAME, new String[] {SetPriorityAction.NAME, AllocateAction.NAME, FreezeAction.NAME}); assertNextActionName("cold", UnfollowAction.NAME, AllocateAction.NAME, new String[] {AllocateAction.NAME, FreezeAction.NAME}); assertNextActionName("cold", UnfollowAction.NAME, FreezeAction.NAME, new String[] {FreezeAction.NAME}); assertNextActionName("cold", UnfollowAction.NAME, null, new String[] {}); - assertNextActionName("cold", SetPriorityAction.NAME, FreezeAction.NAME, new String[]{SetPriorityAction.NAME, FreezeAction.NAME}); - assertNextActionName("cold", SetPriorityAction.NAME, AllocateAction.NAME, - new String[]{SetPriorityAction.NAME, AllocateAction.NAME}); - assertNextActionName("cold", SetPriorityAction.NAME, null, new String[] { SetPriorityAction.NAME }); - assertNextActionName("cold", SetPriorityAction.NAME, null, new String[] {}); + assertNextActionName("cold", AllocateAction.NAME, null, new String[] { AllocateAction.NAME }); assertNextActionName("cold", AllocateAction.NAME, null, new String[] {}); assertNextActionName("cold", AllocateAction.NAME, null, new String[] {}); assertNextActionName("cold", AllocateAction.NAME, FreezeAction.NAME, FreezeAction.NAME); + assertNextActionName("cold", FreezeAction.NAME, null); assertNextActionName("cold", FreezeAction.NAME, null, AllocateAction.NAME); @@ -415,6 +424,7 @@ public void testGetNextActionName() { // Delete Phase assertNextActionName("delete", DeleteAction.NAME, null, new String[] {}); assertNextActionName("delete", DeleteAction.NAME, null, new String[] { DeleteAction.NAME }); + assertInvalidAction("delete", "foo", new String[] { DeleteAction.NAME }); assertInvalidAction("delete", AllocateAction.NAME, new String[] { DeleteAction.NAME }); assertInvalidAction("delete", ForceMergeAction.NAME, new String[] { DeleteAction.NAME }); @@ -467,6 +477,8 @@ private ConcurrentMap convertActionNamesToActions(Strin return new FreezeAction(); case SetPriorityAction.NAME: return new SetPriorityAction(0); + case UnfollowAction.NAME: + return new UnfollowAction(); } return new DeleteAction(); }).collect(Collectors.toConcurrentMap(LifecycleAction::getWriteableName, Function.identity()));