[ILM] Add date setting to calculate index age (#46561)

* [ILM] Add date setting to calculate index age

Add the `index.lifecycle.origination_date` to allow users to configure a
custom date that'll be used to calculate the index age for the phase
transmissions (as opposed to the default index creation date).

This could be useful for users to create an index with an "older"
origination date when indexing old data.

Relates to #42449.

* [ILM] Don't override creation date on policy init

The initial approach we took was to override the lifecycle creation date
if the `index.lifecycle.origination_date` setting was set. This had the
disadvantage of the user not being able to update the `origination_date`
anymore once set.

This commit changes the way we makes use of the
`index.lifecycle.origination_date` setting by checking its value when
we calculate the index age (ie. at "read time") and, in case it's not
set, default to the index creation date.

* Make origination date setting index scope dynamic

* Document orignation date setting in ilm settings

docs/reference/ilm/policy-definitions.asciidoc

@@ -60,10 +60,12 @@ phase after one day. Until then, the index is in a waiting state. After
 moving into the warm phase, it will wait until 30 days have elapsed before
 moving to the delete phase and deleting the index.
 
-`min_age` is usually the time elapsed from the time the index is created. If the
-index is rolled over, then `min_age` is the time elapsed from the time the index
-is rolled over. The intention here is to execute following phases and actions
-relative to when data was written last to a rolled over index.
+`min_age` is usually the time elapsed from the time the index is created, unless
+the `index.lifecycle.origination_date` index setting is configured, in which
+case the `min_age` will be the time elapsed since that specified date. If the
+index is rolled over, then `min_age` is the time elapsed from the time the
+index is rolled over. The intention here is to execute following phases and
+actions relative to when data was written last to a rolled over index.
 
 The previous phase's actions must complete before {ilm} will check `min_age` and
 transition into the next phase. By default, {ilm} checks for indices that meet

docs/reference/settings/ilm-settings.asciidoc

@@ -17,3 +17,8 @@ information about rollover, see <<using-policies-rollover>>.
 `indices.lifecycle.poll_interval`::
 (<<time-units, time units>>) How often {ilm} checks for indices that meet policy
 criteria. Defaults to `10m`.
+
+`index.lifecycle.origination_date`::
+The timestamp that will be used to calculate the index age for its phase
+transitions. This allows the users to create an index containing old data and
+use the original creation date of the old data to calculate the index age.

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -283,7 +283,7 @@ public Iterator<Setting<Integer>> settings() {
     private final long mappingVersion;
 
     private final long settingsVersion;
-    
+
     private final long aliasesVersion;
 
     private final long[] primaryTerms;
@@ -1042,25 +1042,25 @@ public Builder mappingVersion(final long mappingVersion) {
             this.mappingVersion = mappingVersion;
             return this;
         }
-        
+
         public long settingsVersion() {
             return settingsVersion;
         }
-        
+
         public Builder settingsVersion(final long settingsVersion) {
             this.settingsVersion = settingsVersion;
             return this;
         }
-        
+
         public long aliasesVersion() {
             return aliasesVersion;
         }
-        
+
         public Builder aliasesVersion(final long aliasesVersion) {
             this.aliasesVersion = aliasesVersion;
             return this;
         }
-        
+
         /**
          * returns the primary term for the given shard.
          * See {@link IndexMetaData#primaryTerm(int)} for more information.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java

@@ -22,7 +22,7 @@ public final class InitializePolicyContextStep extends ClusterStateActionStep {
     public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
     private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);
 
-    public InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
+    InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
         super(key, nextStepKey);
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java

@@ -17,6 +17,7 @@ public class LifecycleSettings {
     public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval";
     public static final String LIFECYCLE_NAME = "index.lifecycle.name";
     public static final String LIFECYCLE_INDEXING_COMPLETE = "index.lifecycle.indexing_complete";
+    public static final String LIFECYCLE_ORIGINATION_DATE = "index.lifecycle.origination_date";
 
     public static final String SLM_HISTORY_INDEX_ENABLED = "slm.history_index_enabled";
     public static final String SLM_RETENTION_SCHEDULE = "slm.retention_schedule";
@@ -29,6 +30,8 @@ public class LifecycleSettings {
         Setting.Property.Dynamic, Setting.Property.IndexScope);
     public static final Setting<Boolean> LIFECYCLE_INDEXING_COMPLETE_SETTING = Setting.boolSetting(LIFECYCLE_INDEXING_COMPLETE, false,
         Setting.Property.Dynamic, Setting.Property.IndexScope);
+    public static final Setting<Long> LIFECYCLE_ORIGINATION_DATE_SETTING =
+        Setting.longSetting(LIFECYCLE_ORIGINATION_DATE, -1, -1, Setting.Property.Dynamic, Setting.Property.IndexScope);
 
     public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(SLM_HISTORY_INDEX_ENABLED, true,
         Setting.Property.NodeScope);

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -138,6 +138,7 @@ public List<Setting<?>> getSettings() {
         return Arrays.asList(
             LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING,
             LifecycleSettings.LIFECYCLE_NAME_SETTING,
+            LifecycleSettings.LIFECYCLE_ORIGINATION_DATE_SETTING,
             LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING,
             RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING,
             LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING,

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java

@@ -50,6 +50,7 @@
 
 import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE;
 import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_ORIGINATION_DATE;
 
 public class IndexLifecycleRunner {
     private static final Logger logger = LogManager.getLogger(IndexLifecycleRunner.class);
@@ -73,11 +74,12 @@ public IndexLifecycleRunner(PolicyStepsRegistry stepRegistry, ClusterService clu
      */
     boolean isReadyToTransitionToThisPhase(final String policy, final IndexMetaData indexMetaData, final String phase) {
         LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
-        if (lifecycleState.getLifecycleDate() == null) {
-            logger.trace("no index creation date has been set yet");
+        Long originationDate = indexMetaData.getSettings().getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L);
+        if (lifecycleState.getLifecycleDate() == null && originationDate == -1L) {
+            logger.trace("no index creation or origination date has been set yet");
             return true;
         }
-        final Long lifecycleDate = lifecycleState.getLifecycleDate();
+        final Long lifecycleDate = originationDate != -1L ? originationDate : lifecycleState.getLifecycleDate();
         assert lifecycleDate != null && lifecycleDate >= 0 : "expected index to have a lifecycle date but it did not";
         final TimeValue after = stepRegistry.getIndexAgeForPhase(policy, phase);
         final long now = nowSupplier.getAsLong();

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java

@@ -41,6 +41,8 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_ORIGINATION_DATE;
+
 public class TransportExplainLifecycleAction
         extends TransportClusterInfoAction<ExplainLifecycleRequest, ExplainLifecycleResponse> {
 
@@ -107,8 +109,9 @@ protected void doMasterOperation(ExplainLifecycleRequest request, String[] concr
                 // If this is requesting only errors, only include indices in the error step or which are using a nonexistent policy
                 if (request.onlyErrors() == false
                     || (ErrorStep.NAME.equals(lifecycleState.getStep()) || indexLifecycleService.policyExists(policyName) == false)) {
+                    Long originationDate = idxSettings.getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L);
                     indexResponse = IndexLifecycleExplainResponse.newManagedIndexResponse(index, policyName,
-                        lifecycleState.getLifecycleDate(),
+                        originationDate != -1L ? originationDate : lifecycleState.getLifecycleDate(),
                         lifecycleState.getPhase(),
                         lifecycleState.getAction(),
                         lifecycleState.getStep(),

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java

@@ -59,6 +59,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.client.Requests.clusterHealthRequest;
@@ -186,7 +187,7 @@ public void testSingleNodeCluster() throws Exception {
     public void testExplainExecution() throws Exception {
         // start node
         logger.info("Starting server1");
-        final String server_1 = internalCluster().startNode();
+        internalCluster().startNode();
         logger.info("Creating lifecycle [test_lifecycle]");
         PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
         PutLifecycleAction.Response putLifecycleResponse = client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get();
@@ -205,15 +206,39 @@ public void testExplainExecution() throws Exception {
             .actionGet();
         assertAcked(createIndexResponse);
 
+        // using AtomicLong only to extract a value from a lambda rather than the more traditional atomic update use-case
+        AtomicLong originalLifecycleDate = new AtomicLong();
         {
             PhaseExecutionInfo expectedExecutionInfo = new PhaseExecutionInfo(lifecyclePolicy.getName(), mockPhase, 1L, actualModifiedDate);
             assertBusy(() -> {
-                ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
-                ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
-                assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
-                IndexLifecycleExplainResponse indexResponse = explainResponse.getIndexResponses().get("test");
+                IndexLifecycleExplainResponse indexResponse = executeExplainRequestAndGetTestIndexResponse();
                 assertThat(indexResponse.getStep(), equalTo("observable_cluster_state_action"));
                 assertThat(indexResponse.getPhaseExecutionInfo(), equalTo(expectedExecutionInfo));
+                originalLifecycleDate.set(indexResponse.getLifecycleDate());
+            });
+        }
+
+        // set the origination date setting to an older value
+        client().admin().indices().prepareUpdateSettings("test")
+            .setSettings(Collections.singletonMap(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, 1000L)).get();
+
+        {
+            assertBusy(() -> {
+                IndexLifecycleExplainResponse indexResponse = executeExplainRequestAndGetTestIndexResponse();
+                assertThat("The configured origination date dictates the lifecycle date",
+                    indexResponse.getLifecycleDate(), equalTo(1000L));
+            });
+        }
+
+        // set the origination date setting to null
+        client().admin().indices().prepareUpdateSettings("test")
+            .setSettings(Collections.singletonMap(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, null)).get();
+
+        {
+            assertBusy(() -> {
+                IndexLifecycleExplainResponse indexResponse = executeExplainRequestAndGetTestIndexResponse();
+                assertThat("Without the origination date, the index create date should dictate the lifecycle date",
+                    indexResponse.getLifecycleDate(), equalTo(originalLifecycleDate.get()));
             });
         }
 
@@ -224,17 +249,21 @@ public void testExplainExecution() throws Exception {
         {
             PhaseExecutionInfo expectedExecutionInfo = new PhaseExecutionInfo(lifecyclePolicy.getName(), null, 1L, actualModifiedDate);
             assertBusy(() -> {
-                ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
-                ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
-                assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
-                IndexLifecycleExplainResponse indexResponse = explainResponse.getIndexResponses().get("test");
+                IndexLifecycleExplainResponse indexResponse = executeExplainRequestAndGetTestIndexResponse();
                 assertThat(indexResponse.getPhase(), equalTo(TerminalPolicyStep.COMPLETED_PHASE));
                 assertThat(indexResponse.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
                 assertThat(indexResponse.getPhaseExecutionInfo(), equalTo(expectedExecutionInfo));
             });
         }
     }
 
+    private IndexLifecycleExplainResponse executeExplainRequestAndGetTestIndexResponse() throws ExecutionException, InterruptedException {
+        ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
+        ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
+        assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
+        return explainResponse.getIndexResponses().get("test");
+    }
+
     public void testMasterDedicatedDataDedicated() throws Exception {
         settings = Settings.builder().put(settings).put("index.lifecycle.test.complete", true).build();
         // start master node

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java

@@ -1213,6 +1213,18 @@ public void testIsReadyToTransition() {
         now.set(Long.MAX_VALUE);
         assertTrue("index should be able to transition past phase's age",
             runner.isReadyToTransitionToThisPhase(policyName, indexMetaData, "phase"));
+
+        // Come back to the "present"
+        now.set(5L);
+        indexMetaData = IndexMetaData.builder(indexMetaData)
+            .settings(Settings.builder()
+                .put(indexMetaData.getSettings())
+                .put(LifecycleSettings.LIFECYCLE_ORIGINATION_DATE, 3L)
+                .build())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .build();
+        assertTrue("index should be able to transition due to the origination date indicating it's old enough",
+            runner.isReadyToTransitionToThisPhase(policyName, indexMetaData, "phase"));
     }