Skip to content

Commit

Permalink
[ILM] Add date setting to calculate index age
Browse files Browse the repository at this point in the history 
Add the `index.lifecycle.origination_date` to allow users to configure a
custom date that'll be used to calculate the index age for the phase
transmissions (as opposed to the default index creation date).

This could be useful for users to create an index with an "older"
origination date when indexing old data.

Relates to elastic#42449.
  • Loading branch information
@andreidan
andreidan committed Sep 10, 2019
1 parent 6e2bb6f commit 9bbdec9
Show file tree
Hide file tree
Showing 8 changed files with 77 additions and 12 deletions.
10 changes: 6 additions & 4 deletions docs/reference/ilm/policy-definitions.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,12 @@ phase after one day. Until then, the index is in a waiting state. After
moving into the warm phase, it will wait until 30 days have elapsed before
moving to the delete phase and deleting the index.

`min_age` is usually the time elapsed from the time the index is created. If the
index is rolled over, then `min_age` is the time elapsed from the time the index
is rolled over. The intention here is to execute following phases and actions
relative to when data was written last to a rolled over index.
`min_age` is usually the time elapsed from the time the index is created, unless
the `index.lifecycle.origination_date` index setting is configured, in which
case the `min_age` will be the time elapsed since that specified date. If the
index is rolled over, then `min_age` is the time elapsed from the time the
index is rolled over. The intention here is to execute following phases and
actions relative to when data was written last to a rolled over index.

The previous phase's actions must complete before {ilm} will check `min_age` and
transition into the next phase. By default, {ilm} checks for indices that meet
Expand Down
29 changes: 23 additions & 6 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,8 @@ public Iterator<Setting<Integer>> settings() {
public static final String SETTING_VERSION_UPGRADED = "index.version.upgraded";
public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string";
public static final String SETTING_CREATION_DATE = "index.creation_date";
public static final String SETTING_LIFECYCLE_ORIGINATION_DATE = "index.lifecycle.origination_date";

/**
* The user provided name for an index. This is the plain string provided by the user when the index was created.
* It might still contain date math expressions etc. (added in 5.0)
Expand All @@ -217,6 +219,7 @@ public Iterator<Setting<Integer>> settings() {
public static final Setting<Integer> INDEX_PRIORITY_SETTING =
Setting.intSetting("index.priority", 1, 0, Property.Dynamic, Property.IndexScope);
public static final String SETTING_CREATION_DATE_STRING = "index.creation_date_string";
public static final String SETTING_LIFECYLE_ORIGINATION_DATE_STRING = "index.lifecyle.origination_date_string";
public static final String SETTING_INDEX_UUID = "index.uuid";
public static final String SETTING_DATA_PATH = "index.data_path";
public static final Setting<String> INDEX_DATA_PATH_SETTING =
Expand Down Expand Up @@ -283,7 +286,7 @@ public Iterator<Setting<Integer>> settings() {
private final long mappingVersion;

private final long settingsVersion;

private final long aliasesVersion;

private final long[] primaryTerms;
Expand Down Expand Up @@ -439,6 +442,10 @@ public long getCreationDate() {
return settings.getAsLong(SETTING_CREATION_DATE, -1L);
}

public long getOriginationDate() {
return settings.getAsLong(SETTING_LIFECYCLE_ORIGINATION_DATE, -1L);
}

public State getState() {
return this.state;
}
Expand Down Expand Up @@ -954,6 +961,11 @@ public Builder creationDate(long creationDate) {
return this;
}

public Builder lifecycleOriginationDate(long originationDate) {
settings = Settings.builder().put(settings).put(SETTING_LIFECYCLE_ORIGINATION_DATE, originationDate).build();
return this;
}

public Builder settings(Settings.Builder settings) {
return settings(settings.build());
}
Expand Down Expand Up @@ -1042,25 +1054,25 @@ public Builder mappingVersion(final long mappingVersion) {
this.mappingVersion = mappingVersion;
return this;
}

public long settingsVersion() {
return settingsVersion;
}

public Builder settingsVersion(final long settingsVersion) {
this.settingsVersion = settingsVersion;
return this;
}

public long aliasesVersion() {
return aliasesVersion;
}

public Builder aliasesVersion(final long aliasesVersion) {
this.aliasesVersion = aliasesVersion;
return this;
}

/**
* returns the primary term for the given shard.
* See {@link IndexMetaData#primaryTerm(int)} for more information.
Expand Down Expand Up @@ -1440,6 +1452,11 @@ public static Settings addHumanReadableSettings(Settings settings) {
ZonedDateTime creationDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(creationDate), ZoneOffset.UTC);
builder.put(SETTING_CREATION_DATE_STRING, creationDateTime.toString());
}
Long lifecycleOriginationDate = settings.getAsLong(SETTING_LIFECYCLE_ORIGINATION_DATE, null);
if (lifecycleOriginationDate != null) {
ZonedDateTime originationDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(lifecycleOriginationDate), ZoneOffset.UTC);
builder.put(SETTING_LIFECYLE_ORIGINATION_DATE_STRING, originationDateTime.toString());
}
return builder.build();
}

Expand Down
4 changes: 4 additions & 0 deletions server/src/test/java/org/elasticsearch/cluster/metadata/HumanReadableIndexSettingsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,12 @@ public void testHumanReadableSettings() {
Version versionCreated = randomVersion(random());
Version versionUpgraded = randomVersion(random());
long created = System.currentTimeMillis();
long lifecycleOriginationDate = System.currentTimeMillis();
Settings testSettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, versionCreated)
.put(IndexMetaData.SETTING_VERSION_UPGRADED, versionUpgraded)
.put(IndexMetaData.SETTING_CREATION_DATE, created)
.put(IndexMetaData.SETTING_LIFECYCLE_ORIGINATION_DATE, lifecycleOriginationDate)
.build();

Settings humanSettings = IndexMetaData.addHumanReadableSettings(testSettings);
Expand All @@ -46,5 +48,7 @@ public void testHumanReadableSettings() {
assertEquals(versionUpgraded.toString(), humanSettings.get(IndexMetaData.SETTING_VERSION_UPGRADED_STRING, null));
ZonedDateTime creationDate = ZonedDateTime.ofInstant(Instant.ofEpochMilli(created), ZoneOffset.UTC);
assertEquals(creationDate.toString(), humanSettings.get(IndexMetaData.SETTING_CREATION_DATE_STRING, null));
ZonedDateTime originationDate = ZonedDateTime.ofInstant(Instant.ofEpochMilli(lifecycleOriginationDate), ZoneOffset.UTC);
assertEquals(originationDate.toString(), humanSettings.get(IndexMetaData.SETTING_LIFECYLE_ORIGINATION_DATE_STRING));
}
}
3 changes: 2 additions & 1 deletion ...ugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public final class InitializePolicyContextStep extends ClusterStateActionStep {
public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);

public InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
super(key, nextStepKey);
}

Expand All @@ -44,6 +44,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {

LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
newCustomData.setIndexCreationDate(indexMetaData.getCreationDate());
newCustomData.setIndexOriginationDate(indexMetaData.getOriginationDate());
newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData()).put(IndexMetaData
.builder(indexMetaData)
.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())));
Expand Down
19 changes: 18 additions & 1 deletion ...k/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class LifecycleExecutionState {
private static final String ACTION = "action";
private static final String STEP = "step";
private static final String INDEX_CREATION_DATE = "creation_date";
private static final String INDEX_ORIGINATION_DATE = "origination_date";
private static final String PHASE_TIME = "phase_time";
private static final String ACTION_TIME = "action_time";
private static final String STEP_TIME = "step_time";
Expand Down Expand Up @@ -118,6 +119,15 @@ static LifecycleExecutionState fromCustomMetadata(Map<String, String> customData
e, INDEX_CREATION_DATE, customData.get(INDEX_CREATION_DATE));
}
}
if (customData.containsKey(INDEX_ORIGINATION_DATE)) {
try {
builder.setIndexOriginationDate(Long.parseLong(customData.get(INDEX_ORIGINATION_DATE)));
} catch (NumberFormatException e) {
throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
e, INDEX_ORIGINATION_DATE, customData.get(INDEX_ORIGINATION_DATE));
}
}

if (customData.containsKey(PHASE_TIME)) {
try {
builder.setPhaseTime(Long.parseLong(customData.get(PHASE_TIME)));
Expand Down Expand Up @@ -259,6 +269,7 @@ public static class Builder {
private Long phaseTime;
private Long actionTime;
private Long stepTime;
private Long indexOriginationDate;

public Builder setPhase(String phase) {
this.phase = phase;
Expand Down Expand Up @@ -310,8 +321,14 @@ public Builder setStepTime(Long stepTime) {
return this;
}

public Builder setIndexOriginationDate(Long originationDate) {
this.indexOriginationDate = originationDate;
return this;
}

public LifecycleExecutionState build() {
return new LifecycleExecutionState(phase, action, step, failedStep, stepInfo, phaseDefinition, indexCreationDate,
return new LifecycleExecutionState(phase, action, step, failedStep, stepInfo, phaseDefinition,
indexOriginationDate != null && indexOriginationDate != -1L ? indexOriginationDate : indexCreationDate,
phaseTime, actionTime, stepTime);
}
}
Expand Down
4 changes: 4 additions & 0 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.xpack.core.scheduler.CronSchedule;

import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_LIFECYCLE_ORIGINATION_DATE;

/**
* Class encapsulating settings related to Index Lifecycle Management X-Pack Plugin
*/
Expand All @@ -29,6 +31,8 @@ public class LifecycleSettings {
Setting.Property.Dynamic, Setting.Property.IndexScope);
public static final Setting<Boolean> LIFECYCLE_INDEXING_COMPLETE_SETTING = Setting.boolSetting(LIFECYCLE_INDEXING_COMPLETE, false,
Setting.Property.Dynamic, Setting.Property.IndexScope);
public static final Setting<Long> LIFECYCLE_ORIGINATION_DATE_SETTING =
Setting.longSetting(SETTING_LIFECYCLE_ORIGINATION_DATE, -1, -1, Setting.Property.IndexScope, Setting.Property.NodeScope);

public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(SLM_HISTORY_INDEX_ENABLED, true,
Setting.Property.NodeScope);
Expand Down
19 changes: 19 additions & 0 deletions ...core/src/test/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStepTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,25 @@ public void testAddCreationDate() {
assertThat(getIndexLifecycleDate(index, newState), equalTo(creationDate));
}

public void testLifecyleOriginationDateSupersedesCreationDate() {
long creationDate = 1L;
long originationDate = 2L;
IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
.settings(settings(Version.CURRENT))
.creationDate(creationDate)
.lifecycleOriginationDate(originationDate)
.numberOfShards(1).numberOfReplicas(0).build();
MetaData metaData = MetaData.builder()
.persistentSettings(settings(Version.CURRENT).build())
.put(IndexMetaData.builder(indexMetadata))
.build();
Index index = indexMetadata.getIndex();
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
InitializePolicyContextStep step = new InitializePolicyContextStep(null, null);
ClusterState newState = step.performAction(index, clusterState);
assertThat(getIndexLifecycleDate(index, newState), equalTo(originationDate));
}

public void testDoNothing() {
long creationDate = randomNonNegativeLong();
LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ public List<Setting<?>> getSettings() {
return Arrays.asList(
LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING,
LifecycleSettings.LIFECYCLE_NAME_SETTING,
LifecycleSettings.LIFECYCLE_ORIGINATION_DATE_SETTING,
LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING,
RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING,
LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING,
Expand Down

0 comments on commit 9bbdec9

Please sign in to comment.