From 1944320e3a9ee0d183945146dd274df8bfd90bb2 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 4 Dec 2018 13:47:42 -0700 Subject: [PATCH 1/8] Add setting to bypass Rollover action Adds a setting that indicates that an index is done indexing, set by ILM when the Rollover action completes. This indicates that the Rollover action should be skipped in any future invocations, as long as the index is no longer the write index for its alias. This enables 1) an index with a policy that involves the Rollover action to have the policy removed and switched to another one without use of the move-to-step API, and 2) integrations with Beats and CCR. --- .../indexlifecycle/LifecycleSettings.java | 3 + .../core/indexlifecycle/RolloverAction.java | 16 +++++- .../core/indexlifecycle/RolloverStep.java | 11 ++++ .../UpdateRolloverLifecycleDateStep.java | 45 +++++++++++---- .../WaitForRolloverReadyStep.java | 18 ++++++ .../indexlifecycle/RolloverActionTests.java | 8 ++- .../indexlifecycle/RolloverStepTests.java | 28 +++++++++ .../UpdateRolloverLifecycleDateStepTests.java | 33 ++++++++++- .../WaitForRolloverReadyStepTests.java | 57 +++++++++++++++++++ .../TimeSeriesLifecycleActionsIT.java | 2 + .../xpack/indexlifecycle/IndexLifecycle.java | 1 + 11 files changed, 202 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java index 4f8eb339db7e8..9d6002f685692 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java @@ -14,9 +14,12 @@ public class LifecycleSettings { public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval"; public static final String LIFECYCLE_NAME = "index.lifecycle.name"; + public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete"; public static final Setting LIFECYCLE_POLL_INTERVAL_SETTING = Setting.positiveTimeSetting(LIFECYCLE_POLL_INTERVAL, TimeValue.timeValueMinutes(10), Setting.Property.Dynamic, Setting.Property.NodeScope); public static final Setting LIFECYCLE_NAME_SETTING = Setting.simpleString(LIFECYCLE_NAME, Setting.Property.Dynamic, Setting.Property.IndexScope); + public static final Setting LIFECYCLE_INDEXING_COMPLETE_SETTING = Setting.boolSetting(LIFECYCLE_INDEXING_COMPLETE, false, + Setting.Property.Dynamic, Setting.Property.IndexScope); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java index d6b762966a944..ec5a5c157b2c9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java @@ -6,11 +6,13 @@ package org.elasticsearch.xpack.core.indexlifecycle; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -29,6 +31,7 @@ */ public class RolloverAction implements LifecycleAction { public static final String NAME = "rollover"; + public static final String INDEXING_COMPLETE_STEP_NAME = "set-indexing-complete"; public static final ParseField MAX_SIZE_FIELD = new ParseField("max_size"); public static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs"); public static final ParseField MAX_AGE_FIELD = new ParseField("max_age"); @@ -132,15 +135,21 @@ public boolean isSafeAction() { @Override public List toSteps(Client client, String phase, Step.StepKey nextStepKey) { + Settings indexingComplete = Settings.builder().put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true).build(); + StepKey waitForRolloverReadyStepKey = new StepKey(phase, NAME, WaitForRolloverReadyStep.NAME); StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME); StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME); + StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME); WaitForRolloverReadyStep waitForRolloverReadyStep = new WaitForRolloverReadyStep(waitForRolloverReadyStepKey, rolloverStepKey, client, maxSize, maxAge, maxDocs); RolloverStep rolloverStep = new RolloverStep(rolloverStepKey, updateDateStepKey, client); - UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, nextStepKey); - return Arrays.asList(waitForRolloverReadyStep, rolloverStep, updateDateStep); + UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, setIndexingCompleteStepKey, + System::currentTimeMillis); + UpdateSettingsStep setIndexingCompleteStep = new UpdateSettingsStep(setIndexingCompleteStepKey, nextStepKey, + client, indexingComplete); + return Arrays.asList(waitForRolloverReadyStep, rolloverStep, updateDateStep, setIndexingCompleteStep); } @Override @@ -148,7 +157,8 @@ public List toStepKeys(String phase) { StepKey rolloverReadyStepKey = new StepKey(phase, NAME, WaitForRolloverReadyStep.NAME); StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME); StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME); - return Arrays.asList(rolloverReadyStepKey, rolloverStepKey, updateDateStepKey); + StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME); + return Arrays.asList(rolloverReadyStepKey, rolloverStepKey, updateDateStepKey, setIndexingCompleteStepKey); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStep.java index 1bdbea03df964..f501c27d8c40e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStep.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.indexlifecycle; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.client.Client; @@ -19,6 +21,8 @@ * Unconditionally rolls over an index using the Rollover API. */ public class RolloverStep extends AsyncActionStep { + private static final Logger logger = LogManager.getLogger(RolloverStep.class); + public static final String NAME = "attempt-rollover"; public RolloverStep(StepKey key, StepKey nextStepKey, Client client) { @@ -27,6 +31,13 @@ public RolloverStep(StepKey key, StepKey nextStepKey, Client client) { @Override public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener) { + boolean indexingComplete = LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.get(indexMetaData.getSettings()); + if (indexingComplete) { + logger.trace(indexMetaData.getIndex() + " has lifecycle complete set, skipping " + RolloverStep.NAME); + listener.onResponse(true); + return; + } + String rolloverAlias = RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.get(indexMetaData.getSettings()); if (Strings.isNullOrEmpty(rolloverAlias)) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java index 6f5160fa13a20..daa0399bc2aa3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.indexlifecycle; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -12,6 +14,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.index.Index; +import java.util.function.LongSupplier; + import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; /** @@ -19,30 +23,47 @@ * Used so that the "age" of an index doesn't get reset on rollover. */ public class UpdateRolloverLifecycleDateStep extends ClusterStateActionStep { + private static final Logger logger = LogManager.getLogger(UpdateRolloverLifecycleDateStep.class); public static final String NAME = "update-rollover-lifecycle-date"; - public UpdateRolloverLifecycleDateStep(StepKey key, StepKey nextStepKey) { + private final LongSupplier fallbackTimeSupplier; + + public UpdateRolloverLifecycleDateStep(StepKey key, StepKey nextStepKey, LongSupplier fallbackTimeSupplier) { super(key, nextStepKey); + this.fallbackTimeSupplier = fallbackTimeSupplier; } @Override public ClusterState performAction(Index index, ClusterState currentState) { IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index); - // find the newly created index from the rollover and fetch its index.creation_date - String rolloverAlias = RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.get(indexMetaData.getSettings()); - if (Strings.isNullOrEmpty(rolloverAlias)) { - throw new IllegalStateException("setting [" + RolloverAction.LIFECYCLE_ROLLOVER_ALIAS - + "] is not set on index [" + indexMetaData.getIndex().getName() + "]"); - } - RolloverInfo rolloverInfo = indexMetaData.getRolloverInfos().get(rolloverAlias); - if (rolloverInfo == null) { - throw new IllegalStateException("no rollover info found for [" + indexMetaData.getIndex().getName() + "] with alias [" + - rolloverAlias + "], the index has not yet rolled over with that alias"); + + long newIndexTime; + + boolean indexingComplete = LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.get(indexMetaData.getSettings()); + if (indexingComplete) { + logger.trace(indexMetaData.getIndex() + " has lifecycle complete set, skipping " + UpdateRolloverLifecycleDateStep.NAME); + + // The index won't have RolloverInfo if this is a Following index and indexing_complete was set by CCR, + // so just use the current time. + newIndexTime = fallbackTimeSupplier.getAsLong(); + } else { + // find the newly created index from the rollover and fetch its index.creation_date + String rolloverAlias = RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.get(indexMetaData.getSettings()); + if (Strings.isNullOrEmpty(rolloverAlias)) { + throw new IllegalStateException("setting [" + RolloverAction.LIFECYCLE_ROLLOVER_ALIAS + + "] is not set on index [" + indexMetaData.getIndex().getName() + "]"); + } + RolloverInfo rolloverInfo = indexMetaData.getRolloverInfos().get(rolloverAlias); + if (rolloverInfo == null) { + throw new IllegalStateException("no rollover info found for [" + indexMetaData.getIndex().getName() + "] with alias [" + + rolloverAlias + "], the index has not yet rolled over with that alias"); + } + newIndexTime = rolloverInfo.getTime(); } LifecycleExecutionState.Builder newLifecycleState = LifecycleExecutionState .builder(LifecycleExecutionState.fromIndexMetadata(indexMetaData)); - newLifecycleState.setIndexCreationDate(rolloverInfo.getTime()); + newLifecycleState.setIndexCreationDate(newIndexTime); IndexMetaData.Builder newIndexMetadata = IndexMetaData.builder(indexMetaData); newIndexMetadata.putCustom(ILM_CUSTOM_METADATA_KEY, newLifecycleState.build().asMap()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStep.java index fa9e59985071a..03bf905290ffd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStep.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.core.indexlifecycle; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.client.Client; @@ -24,6 +26,7 @@ * Waits for at least one rollover condition to be satisfied, using the Rollover API's dry_run option. */ public class WaitForRolloverReadyStep extends AsyncWaitStep { + private static final Logger logger = LogManager.getLogger(WaitForRolloverReadyStep.class); public static final String NAME = "check-rollover-ready"; @@ -57,6 +60,21 @@ public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) { return; } + boolean indexingComplete = LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.get(indexMetaData.getSettings()); + if (indexingComplete) { + logger.trace(indexMetaData.getIndex() + " has lifecycle complete set, skipping " + WaitForRolloverReadyStep.NAME); + Boolean isWriteIndex = indexMetaData.getAliases().get(rolloverAlias).writeIndex(); + if (Boolean.TRUE.equals(isWriteIndex)) { + listener.onFailure(new IllegalStateException(String.format(Locale.ROOT, + "index [%s] has [%s] set to [true], but is still the write index for alias [%s]", + indexMetaData.getIndex().getName(), LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, rolloverAlias))); + return; + } + + listener.onResponse(true, new WaitForRolloverReadyStep.EmptyInfo()); + return; + } + RolloverRequest rolloverRequest = new RolloverRequest(rolloverAlias, null); rolloverRequest.dryRun(true); if (maxAge != null) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java index 55cbad7144b4f..d32d9de0ce619 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java @@ -80,21 +80,25 @@ public void testToSteps() { randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(3, steps.size()); + assertEquals(4, steps.size()); StepKey expectedFirstStepKey = new StepKey(phase, RolloverAction.NAME, WaitForRolloverReadyStep.NAME); StepKey expectedSecondStepKey = new StepKey(phase, RolloverAction.NAME, RolloverStep.NAME); StepKey expectedThirdStepKey = new StepKey(phase, RolloverAction.NAME, UpdateRolloverLifecycleDateStep.NAME); + StepKey expectedFourthStepKey = new StepKey(phase, RolloverAction.NAME, RolloverAction.INDEXING_COMPLETE_STEP_NAME); WaitForRolloverReadyStep firstStep = (WaitForRolloverReadyStep) steps.get(0); RolloverStep secondStep = (RolloverStep) steps.get(1); UpdateRolloverLifecycleDateStep thirdStep = (UpdateRolloverLifecycleDateStep) steps.get(2); + UpdateSettingsStep fourthStep = (UpdateSettingsStep) steps.get(3); assertEquals(expectedFirstStepKey, firstStep.getKey()); assertEquals(expectedSecondStepKey, secondStep.getKey()); assertEquals(expectedThirdStepKey, thirdStep.getKey()); + assertEquals(expectedFourthStepKey, fourthStep.getKey()); assertEquals(secondStep.getKey(), firstStep.getNextStepKey()); assertEquals(thirdStep.getKey(), secondStep.getNextStepKey()); + assertEquals(fourthStep.getKey(), thirdStep.getNextStepKey()); assertEquals(action.getMaxSize(), firstStep.getMaxSize()); assertEquals(action.getMaxAge(), firstStep.getMaxAge()); assertEquals(action.getMaxDocs(), firstStep.getMaxDocs()); - assertEquals(nextStepKey, thirdStep.getNextStepKey()); + assertEquals(nextStepKey, fourthStep.getNextStepKey()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStepTests.java index d31643e51959b..5f65b581e00b3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStepTests.java @@ -126,6 +126,34 @@ public void onFailure(Exception e) { Mockito.verify(indicesClient, Mockito.only()).rolloverIndex(Mockito.any(), Mockito.any()); } + public void testPerformActionWithIndexingComplete() { + String alias = randomAlphaOfLength(5); + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .putAlias(AliasMetaData.builder(alias)) + .settings(settings(Version.CURRENT) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + + RolloverStep step = createRandomInstance(); + + SetOnce actionCompleted = new SetOnce<>(); + step.performAction(indexMetaData, null, new AsyncActionStep.Listener() { + + @Override + public void onResponse(boolean complete) { + actionCompleted.set(complete); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Unexpected method call", e); + } + }); + + assertEquals(true, actionCompleted.get()); + } + public void testPerformActionFailure() { String alias = randomAlphaOfLength(5); IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java index 9db45c1b59b2f..f8937535f1ba4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; import java.util.Collections; +import java.util.function.LongSupplier; import static org.hamcrest.Matchers.equalTo; @@ -23,9 +24,13 @@ public class UpdateRolloverLifecycleDateStepTests extends AbstractStepTestCase randomNonNegativeLong()); + + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .settings(settings(Version.CURRENT) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .metaData(MetaData.builder() + .put(indexMetaData, false)).build(); + + UpdateRolloverLifecycleDateStep step = createRandomInstanceWithFallbackTime(() -> rolloverTime); + ClusterState newState = step.performAction(indexMetaData.getIndex(), clusterState); + long actualRolloverTime = LifecycleExecutionState + .fromIndexMetadata(newState.metaData().index(indexMetaData.getIndex())) + .getLifecycleDate(); + assertThat(actualRolloverTime, equalTo(rolloverTime)); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStepTests.java index b6bab1a207a14..f7378278ffb63 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForRolloverReadyStepTests.java @@ -173,6 +173,63 @@ public void onFailure(Exception e) { Mockito.verify(indicesClient, Mockito.only()).rolloverIndex(Mockito.any(), Mockito.any()); } + public void testPerformActionWithIndexingComplete() { + String alias = randomAlphaOfLength(5); + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .putAlias(AliasMetaData.builder(alias).writeIndex(randomFrom(false, null))) + .settings(settings(Version.CURRENT) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + + WaitForRolloverReadyStep step = createRandomInstance(); + + SetOnce conditionsMet = new SetOnce<>(); + step.evaluateCondition(indexMetaData, new AsyncWaitStep.Listener() { + + @Override + public void onResponse(boolean complete, ToXContentObject infomationContext) { + conditionsMet.set(complete); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Unexpected method call", e); + } + }); + + assertEquals(true, conditionsMet.get()); + } + + public void testPerformActionWithIndexingCompleteStillWriteIndex() { + String alias = randomAlphaOfLength(5); + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .putAlias(AliasMetaData.builder(alias).writeIndex(true)) + .settings(settings(Version.CURRENT) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) + .put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true)) + .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build(); + + WaitForRolloverReadyStep step = createRandomInstance(); + + SetOnce correctFailureCalled = new SetOnce<>(); + step.evaluateCondition(indexMetaData, new AsyncWaitStep.Listener() { + + @Override + public void onResponse(boolean complete, ToXContentObject infomationContext) { + throw new AssertionError("Should have failed with indexing_complete but index is not write index"); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof IllegalStateException); + correctFailureCalled.set(true); + } + }); + + assertEquals(true, correctFailureCalled.get()); + } + public void testPerformActionNotComplete() { String alias = randomAlphaOfLength(5); IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index adf6ab8972bab..285687887c3a6 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.ForceMergeAction; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy; +import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings; import org.elasticsearch.xpack.core.indexlifecycle.Phase; import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction; import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; @@ -230,6 +231,7 @@ public void testRolloverAction() throws Exception { index(client(), originalIndex, "_id", "foo", "bar"); assertBusy(() -> assertTrue(indexExists(secondIndex))); assertBusy(() -> assertTrue(indexExists(originalIndex))); + assertBusy(() -> assertEquals("true", getOnlyIndexSettings(originalIndex).get(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE))); } public void testRolloverAlreadyExists() throws Exception { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java index 1e42846b317d3..27fb52fe2397f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java @@ -120,6 +120,7 @@ public List> getSettings() { return Arrays.asList( LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING, LifecycleSettings.LIFECYCLE_NAME_SETTING, + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING, RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING); } From 6b43803f7acf57662d213b4f54fad64f76fa5b51 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 4 Dec 2018 15:31:17 -0700 Subject: [PATCH 2/8] Remove unused import --- .../elasticsearch/xpack/core/indexlifecycle/RolloverAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java index ec5a5c157b2c9..25346fefa3149 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.core.indexlifecycle; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; From a48a0e667c663adc882f01fac98174e9314c26f6 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 5 Dec 2018 17:46:28 -0700 Subject: [PATCH 3/8] Add integration test --- .../TimeSeriesLifecycleActionsIT.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 285687887c3a6..19e8d80ac1a91 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -234,6 +234,46 @@ public void testRolloverAction() throws Exception { assertBusy(() -> assertEquals("true", getOnlyIndexSettings(originalIndex).get(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE))); } + public void testRolloverActionWithIndexingComplete() throws Exception { + String originalIndex = index + "-000001"; + String secondIndex = index + "-000002"; + createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias")); + + Request updateSettingsRequest = new Request("PUT", "/" + originalIndex + "/_settings"); + updateSettingsRequest.setJsonEntity("{\n" + + " \"settings\": {\n" + + " \"" + LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE + "\": true\n" + + " }\n" + + "}"); + client().performRequest(updateSettingsRequest); + Request updateAliasRequest = new Request("POST", "/_aliases"); + updateAliasRequest.setJsonEntity("{\n" + + " \"actions\": [\n" + + " {\n" + + " \"add\": {\n" + + " \"index\": \"" + originalIndex + "\",\n" + + " \"alias\": \"alias\",\n" + + " \"is_write_index\": false\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"); + client().performRequest(updateAliasRequest); + + // create policy + createNewSingletonPolicy("hot", new RolloverAction(null, null, 1L)); + // update policy on index + updatePolicy(originalIndex, policy); + // index document {"foo": "bar"} to trigger rollover + index(client(), originalIndex, "_id", "foo", "bar"); + assertBusy(() -> assertEquals(TerminalPolicyStep.KEY, getStepKeyForIndex(originalIndex))); + assertBusy(() -> assertTrue(indexExists(originalIndex))); + assertBusy(() -> assertFalse(indexExists(secondIndex))); + assertBusy(() -> assertEquals("true", getOnlyIndexSettings(originalIndex).get(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE))); + } + public void testRolloverAlreadyExists() throws Exception { String originalIndex = index + "-000001"; String secondIndex = index + "-000002"; From 68e15044492f16efd7d2089200c88e24254534c9 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 6 Dec 2018 16:18:10 -0700 Subject: [PATCH 4/8] First draft of docs --- .../ilm/using-policies-rollover.asciidoc | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/reference/ilm/using-policies-rollover.asciidoc b/docs/reference/ilm/using-policies-rollover.asciidoc index 3a51fcecd790b..25fb71346aedc 100644 --- a/docs/reference/ilm/using-policies-rollover.asciidoc +++ b/docs/reference/ilm/using-policies-rollover.asciidoc @@ -124,3 +124,25 @@ suffix number for each subsequent index. When the rollover is performed, the newly-created index is set as the write index for the rolled over alias. Documents sent to the alias are indexed into the new index, enabling indexing to continue uninterrupted. + +=== After Rollover + +beta[] + +After an index has been rolled over by {ilm}, the +`index.lifecycle.indexing_complete` setting will be set to `true` on the index. +This indicates to {ilm} that this index has already been rolled over, and does +not need to be rolled over again. If you <> +from an index and set it to use another policy, this setting indicates that the +new policy should skip execution of the Rollover action. + +IMPORTANT: If `index.lifecycle.indexing_complete` is set to `true` on an index, +it will not be rolled over by {ilm}, but {ilm} will very that this index is no +longer the write index for the alias specified by +`index.lifecycle.rollover_alias`. If that setting is missing, or if the index is +still the write index for that alias, this index will be moved to the +<>. + +In some cases, `index.lifecycle.indexing_complete` may be set automatically by +other components of Elasticsearch or the Elastic Stack which need to handle +rollover in a particular way. \ No newline at end of file From 3726cbbe1b67cd399eb48913e69251769ccecdcd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 6 Dec 2018 18:25:39 -0700 Subject: [PATCH 5/8] Merge indexing_complete setting into previous step --- .../xpack/core/indexlifecycle/RolloverAction.java | 14 +++----------- .../UpdateRolloverLifecycleDateStep.java | 11 +++++++++-- .../core/indexlifecycle/RolloverActionTests.java | 8 ++------ 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java index 25346fefa3149..bf66dcfdf390c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -30,7 +29,6 @@ */ public class RolloverAction implements LifecycleAction { public static final String NAME = "rollover"; - public static final String INDEXING_COMPLETE_STEP_NAME = "set-indexing-complete"; public static final ParseField MAX_SIZE_FIELD = new ParseField("max_size"); public static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs"); public static final ParseField MAX_AGE_FIELD = new ParseField("max_age"); @@ -134,21 +132,16 @@ public boolean isSafeAction() { @Override public List toSteps(Client client, String phase, Step.StepKey nextStepKey) { - Settings indexingComplete = Settings.builder().put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true).build(); - StepKey waitForRolloverReadyStepKey = new StepKey(phase, NAME, WaitForRolloverReadyStep.NAME); StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME); StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME); - StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME); WaitForRolloverReadyStep waitForRolloverReadyStep = new WaitForRolloverReadyStep(waitForRolloverReadyStepKey, rolloverStepKey, client, maxSize, maxAge, maxDocs); RolloverStep rolloverStep = new RolloverStep(rolloverStepKey, updateDateStepKey, client); - UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, setIndexingCompleteStepKey, + UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, nextStepKey, System::currentTimeMillis); - UpdateSettingsStep setIndexingCompleteStep = new UpdateSettingsStep(setIndexingCompleteStepKey, nextStepKey, - client, indexingComplete); - return Arrays.asList(waitForRolloverReadyStep, rolloverStep, updateDateStep, setIndexingCompleteStep); + return Arrays.asList(waitForRolloverReadyStep, rolloverStep, updateDateStep); } @Override @@ -156,8 +149,7 @@ public List toStepKeys(String phase) { StepKey rolloverReadyStepKey = new StepKey(phase, NAME, WaitForRolloverReadyStep.NAME); StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME); StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME); - StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME); - return Arrays.asList(rolloverReadyStepKey, rolloverStepKey, updateDateStepKey, setIndexingCompleteStepKey); + return Arrays.asList(rolloverReadyStepKey, rolloverStepKey, updateDateStepKey); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java index daa0399bc2aa3..321238b24b164 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import java.util.function.LongSupplier; @@ -66,9 +67,15 @@ public ClusterState performAction(Index index, ClusterState currentState) { newLifecycleState.setIndexCreationDate(newIndexTime); IndexMetaData.Builder newIndexMetadata = IndexMetaData.builder(indexMetaData); + if (indexingComplete == false) { + newIndexMetadata + .settings(Settings.builder().put(indexMetaData.getSettings()).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true)) + .settingsVersion(indexMetaData.getSettingsVersion() + 1); + } newIndexMetadata.putCustom(ILM_CUSTOM_METADATA_KEY, newLifecycleState.build().asMap()); - return ClusterState.builder(currentState).metaData(MetaData.builder(currentState.metaData()) - .put(newIndexMetadata)).build(); + return ClusterState.builder(currentState) + .metaData(MetaData.builder(currentState.metaData()) + .put(newIndexMetadata)).build(); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java index d32d9de0ce619..55cbad7144b4f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java @@ -80,25 +80,21 @@ public void testToSteps() { randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(4, steps.size()); + assertEquals(3, steps.size()); StepKey expectedFirstStepKey = new StepKey(phase, RolloverAction.NAME, WaitForRolloverReadyStep.NAME); StepKey expectedSecondStepKey = new StepKey(phase, RolloverAction.NAME, RolloverStep.NAME); StepKey expectedThirdStepKey = new StepKey(phase, RolloverAction.NAME, UpdateRolloverLifecycleDateStep.NAME); - StepKey expectedFourthStepKey = new StepKey(phase, RolloverAction.NAME, RolloverAction.INDEXING_COMPLETE_STEP_NAME); WaitForRolloverReadyStep firstStep = (WaitForRolloverReadyStep) steps.get(0); RolloverStep secondStep = (RolloverStep) steps.get(1); UpdateRolloverLifecycleDateStep thirdStep = (UpdateRolloverLifecycleDateStep) steps.get(2); - UpdateSettingsStep fourthStep = (UpdateSettingsStep) steps.get(3); assertEquals(expectedFirstStepKey, firstStep.getKey()); assertEquals(expectedSecondStepKey, secondStep.getKey()); assertEquals(expectedThirdStepKey, thirdStep.getKey()); - assertEquals(expectedFourthStepKey, fourthStep.getKey()); assertEquals(secondStep.getKey(), firstStep.getNextStepKey()); assertEquals(thirdStep.getKey(), secondStep.getNextStepKey()); - assertEquals(fourthStep.getKey(), thirdStep.getNextStepKey()); assertEquals(action.getMaxSize(), firstStep.getMaxSize()); assertEquals(action.getMaxAge(), firstStep.getMaxAge()); assertEquals(action.getMaxDocs(), firstStep.getMaxDocs()); - assertEquals(nextStepKey, fourthStep.getNextStepKey()); + assertEquals(nextStepKey, thirdStep.getNextStepKey()); } } From df19da54f4335a1c9c61626ef4b3d409266a212d Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 7 Dec 2018 14:40:05 -0700 Subject: [PATCH 6/8] Revert "Merge indexing_complete setting into previous step" This reverts commit 3726cbbe1b67cd399eb48913e69251769ccecdcd. --- .../xpack/core/indexlifecycle/RolloverAction.java | 14 +++++++++++--- .../UpdateRolloverLifecycleDateStep.java | 11 ++--------- .../core/indexlifecycle/RolloverActionTests.java | 8 ++++++-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java index bf66dcfdf390c..25346fefa3149 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -29,6 +30,7 @@ */ public class RolloverAction implements LifecycleAction { public static final String NAME = "rollover"; + public static final String INDEXING_COMPLETE_STEP_NAME = "set-indexing-complete"; public static final ParseField MAX_SIZE_FIELD = new ParseField("max_size"); public static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs"); public static final ParseField MAX_AGE_FIELD = new ParseField("max_age"); @@ -132,16 +134,21 @@ public boolean isSafeAction() { @Override public List toSteps(Client client, String phase, Step.StepKey nextStepKey) { + Settings indexingComplete = Settings.builder().put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true).build(); + StepKey waitForRolloverReadyStepKey = new StepKey(phase, NAME, WaitForRolloverReadyStep.NAME); StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME); StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME); + StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME); WaitForRolloverReadyStep waitForRolloverReadyStep = new WaitForRolloverReadyStep(waitForRolloverReadyStepKey, rolloverStepKey, client, maxSize, maxAge, maxDocs); RolloverStep rolloverStep = new RolloverStep(rolloverStepKey, updateDateStepKey, client); - UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, nextStepKey, + UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, setIndexingCompleteStepKey, System::currentTimeMillis); - return Arrays.asList(waitForRolloverReadyStep, rolloverStep, updateDateStep); + UpdateSettingsStep setIndexingCompleteStep = new UpdateSettingsStep(setIndexingCompleteStepKey, nextStepKey, + client, indexingComplete); + return Arrays.asList(waitForRolloverReadyStep, rolloverStep, updateDateStep, setIndexingCompleteStep); } @Override @@ -149,7 +156,8 @@ public List toStepKeys(String phase) { StepKey rolloverReadyStepKey = new StepKey(phase, NAME, WaitForRolloverReadyStep.NAME); StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME); StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME); - return Arrays.asList(rolloverReadyStepKey, rolloverStepKey, updateDateStepKey); + StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME); + return Arrays.asList(rolloverReadyStepKey, rolloverStepKey, updateDateStepKey, setIndexingCompleteStepKey); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java index 321238b24b164..daa0399bc2aa3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java @@ -12,7 +12,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import java.util.function.LongSupplier; @@ -67,15 +66,9 @@ public ClusterState performAction(Index index, ClusterState currentState) { newLifecycleState.setIndexCreationDate(newIndexTime); IndexMetaData.Builder newIndexMetadata = IndexMetaData.builder(indexMetaData); - if (indexingComplete == false) { - newIndexMetadata - .settings(Settings.builder().put(indexMetaData.getSettings()).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, true)) - .settingsVersion(indexMetaData.getSettingsVersion() + 1); - } newIndexMetadata.putCustom(ILM_CUSTOM_METADATA_KEY, newLifecycleState.build().asMap()); - return ClusterState.builder(currentState) - .metaData(MetaData.builder(currentState.metaData()) - .put(newIndexMetadata)).build(); + return ClusterState.builder(currentState).metaData(MetaData.builder(currentState.metaData()) + .put(newIndexMetadata)).build(); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java index 55cbad7144b4f..d32d9de0ce619 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java @@ -80,21 +80,25 @@ public void testToSteps() { randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); - assertEquals(3, steps.size()); + assertEquals(4, steps.size()); StepKey expectedFirstStepKey = new StepKey(phase, RolloverAction.NAME, WaitForRolloverReadyStep.NAME); StepKey expectedSecondStepKey = new StepKey(phase, RolloverAction.NAME, RolloverStep.NAME); StepKey expectedThirdStepKey = new StepKey(phase, RolloverAction.NAME, UpdateRolloverLifecycleDateStep.NAME); + StepKey expectedFourthStepKey = new StepKey(phase, RolloverAction.NAME, RolloverAction.INDEXING_COMPLETE_STEP_NAME); WaitForRolloverReadyStep firstStep = (WaitForRolloverReadyStep) steps.get(0); RolloverStep secondStep = (RolloverStep) steps.get(1); UpdateRolloverLifecycleDateStep thirdStep = (UpdateRolloverLifecycleDateStep) steps.get(2); + UpdateSettingsStep fourthStep = (UpdateSettingsStep) steps.get(3); assertEquals(expectedFirstStepKey, firstStep.getKey()); assertEquals(expectedSecondStepKey, secondStep.getKey()); assertEquals(expectedThirdStepKey, thirdStep.getKey()); + assertEquals(expectedFourthStepKey, fourthStep.getKey()); assertEquals(secondStep.getKey(), firstStep.getNextStepKey()); assertEquals(thirdStep.getKey(), secondStep.getNextStepKey()); + assertEquals(fourthStep.getKey(), thirdStep.getNextStepKey()); assertEquals(action.getMaxSize(), firstStep.getMaxSize()); assertEquals(action.getMaxAge(), firstStep.getMaxAge()); assertEquals(action.getMaxDocs(), firstStep.getMaxDocs()); - assertEquals(nextStepKey, thirdStep.getNextStepKey()); + assertEquals(nextStepKey, fourthStep.getNextStepKey()); } } From 8b399c1eb8f50d3c66f2466631d49ae8d5fb36b9 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 7 Dec 2018 15:44:10 -0700 Subject: [PATCH 7/8] Rework docs --- .../ilm/using-policies-rollover.asciidoc | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/reference/ilm/using-policies-rollover.asciidoc b/docs/reference/ilm/using-policies-rollover.asciidoc index 25fb71346aedc..12567403d867f 100644 --- a/docs/reference/ilm/using-policies-rollover.asciidoc +++ b/docs/reference/ilm/using-policies-rollover.asciidoc @@ -125,7 +125,7 @@ When the rollover is performed, the newly-created index is set as the write index for the rolled over alias. Documents sent to the alias are indexed into the new index, enabling indexing to continue uninterrupted. -=== After Rollover +=== Skipping Rollover beta[] @@ -136,6 +136,12 @@ not need to be rolled over again. If you <> from an index and set it to use another policy, this setting indicates that the new policy should skip execution of the Rollover action. +You can also set this setting to `true` manually if you want to indicate that +{ilm} should not roll over a particular index. This is useful if you need to +make an exception to your normal Lifecycle Policy and switching the alias to a +different index by hand, but do not want to remove the index from {ilm} +completely. + IMPORTANT: If `index.lifecycle.indexing_complete` is set to `true` on an index, it will not be rolled over by {ilm}, but {ilm} will very that this index is no longer the write index for the alias specified by @@ -143,6 +149,11 @@ longer the write index for the alias specified by still the write index for that alias, this index will be moved to the <>. -In some cases, `index.lifecycle.indexing_complete` may be set automatically by -other components of Elasticsearch or the Elastic Stack which need to handle -rollover in a particular way. \ No newline at end of file +For example, if you wish to change the name of new indices while retaining +previous data in accordance with your configured policy, you can create the +template for the new index name pattern and the first index with the new name +manually, change the write index of the alias using the <>, and set `index.lifecycle.indexing_complete` to `true` on the old +index to indicate that it does not need to be rolled over. This way, {ilm} will +continue to manage the old index in accordance with its existing policy, as well +as the new one, with no interruption. From 38c6cdf9fbcc5cbfca4be9bf47cd72f92e03672a Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 10 Dec 2018 17:30:00 -0700 Subject: [PATCH 8/8] Typo --- docs/reference/ilm/using-policies-rollover.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ilm/using-policies-rollover.asciidoc b/docs/reference/ilm/using-policies-rollover.asciidoc index 12567403d867f..3cc4271e83dc8 100644 --- a/docs/reference/ilm/using-policies-rollover.asciidoc +++ b/docs/reference/ilm/using-policies-rollover.asciidoc @@ -143,7 +143,7 @@ different index by hand, but do not want to remove the index from {ilm} completely. IMPORTANT: If `index.lifecycle.indexing_complete` is set to `true` on an index, -it will not be rolled over by {ilm}, but {ilm} will very that this index is no +it will not be rolled over by {ilm}, but {ilm} will verify that this index is no longer the write index for the alias specified by `index.lifecycle.rollover_alias`. If that setting is missing, or if the index is still the write index for that alias, this index will be moved to the