Skip to content

Commit

Permalink
Add built-in ILM policies for common user use cases (#76791) (#78278)
Browse files Browse the repository at this point in the history 
This commit adds five built-in ILM policies ranging from 7 days to 365 days of retention. These are
intending to be starting points for a user to use until they switch to using a custom built ILM
policy.
  • Loading branch information
@dakrone
dakrone authored Sep 23, 2021
1 parent 1b4f382 commit 692f694
Show file tree
Hide file tree
Showing 8 changed files with 231 additions and 5 deletions.
37 changes: 37 additions & 0 deletions x-pack/plugin/core/src/main/resources/180-days-default.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"phases": {
"hot": {
"actions": {
"rollover": {
"max_primary_shard_size": "50gb",
"max_age": "30d"
}
}
},
"warm": {
"min_age": "2d",
"actions": {
"shrink": {
"number_of_shards": 1
},
"forcemerge": {
"max_num_segments": 1
}
}
},
"cold": {
"min_age": "30d",
"actions": {}
},
"delete": {
"min_age": "180d",
"actions":{
"delete": {}
}
}
},
"_meta": {
"description": "built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days",
"managed": true
}
}
33 changes: 33 additions & 0 deletions x-pack/plugin/core/src/main/resources/30-days-default.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"phases": {
"hot": {
"actions": {
"rollover": {
"max_primary_shard_size": "50gb",
"max_age": "30d"
}
}
},
"warm": {
"min_age": "2d",
"actions": {
"shrink": {
"number_of_shards": 1
},
"forcemerge": {
"max_num_segments": 1
}
}
},
"delete": {
"min_age": "30d",
"actions":{
"delete": {}
}
}
},
"_meta": {
"description": "built-in ILM policy using the hot and warm phases with a retention of 30 days",
"managed": true
}
}
37 changes: 37 additions & 0 deletions x-pack/plugin/core/src/main/resources/365-days-default.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"phases": {
"hot": {
"actions": {
"rollover": {
"max_primary_shard_size": "50gb",
"max_age": "30d"
}
}
},
"warm": {
"min_age": "2d",
"actions": {
"shrink": {
"number_of_shards": 1
},
"forcemerge": {
"max_num_segments": 1
}
}
},
"cold": {
"min_age": "30d",
"actions": {}
},
"delete": {
"min_age": "365d",
"actions":{
"delete": {}
}
}
},
"_meta": {
"description": "built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days",
"managed": true
}
}
33 changes: 33 additions & 0 deletions x-pack/plugin/core/src/main/resources/7-days-default.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"phases": {
"hot": {
"actions": {
"rollover": {
"max_primary_shard_size": "50gb",
"max_age": "7d"
}
}
},
"warm": {
"min_age": "2d",
"actions": {
"shrink": {
"number_of_shards": 1
},
"forcemerge": {
"max_num_segments": 1
}
}
},
"delete": {
"min_age": "7d",
"actions":{
"delete": {}
}
}
},
"_meta": {
"description": "built-in ILM policy using the hot and warm phases with a retention of 7 days",
"managed": true
}
}
37 changes: 37 additions & 0 deletions x-pack/plugin/core/src/main/resources/90-days-default.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"phases": {
"hot": {
"actions": {
"rollover": {
"max_primary_shard_size": "50gb",
"max_age": "30d"
}
}
},
"warm": {
"min_age": "2d",
"actions": {
"shrink": {
"number_of_shards": 1
},
"forcemerge": {
"max_num_segments": 1
}
}
},
"cold": {
"min_age": "30d",
"actions": {}
},
"delete": {
"min_age": "90d",
"actions":{
"delete": {}
}
}
},
"_meta": {
"description": "built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days",
"managed": true
}
}
1 change: 1 addition & 0 deletions x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ testClusters.all {
setting 'xpack.ml.min_disk_space_off_heap', '200mb'
setting 'indices.lifecycle.history_index_enabled', 'false'
setting 'slm.history_index_enabled', 'false'
setting 'stack.templates.enabled', 'false'
// To spice things up a bit, one of the nodes is not an ML node
nodes.'javaRestTest-0'.setting 'node.roles', '["master","data","ingest"]'

Expand Down
41 changes: 40 additions & 1 deletion x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,36 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
TEMPLATE_VERSION_VARIABLE
);

//////////////////////////////////////////////////////////
// Built in ILM policies for users to use
//////////////////////////////////////////////////////////
public static final String ILM_7_DAYS_POLICY_NAME = "7-days-default";
public static final String ILM_30_DAYS_POLICY_NAME = "30-days-default";
public static final String ILM_90_DAYS_POLICY_NAME = "90-days-default";
public static final String ILM_180_DAYS_POLICY_NAME = "180-days-default";
public static final String ILM_365_DAYS_POLICY_NAME = "365-days-default";

public static final LifecyclePolicyConfig ILM_7_DAYS_POLICY = new LifecyclePolicyConfig(
ILM_7_DAYS_POLICY_NAME,
"/" + ILM_7_DAYS_POLICY_NAME + ".json"
);
public static final LifecyclePolicyConfig ILM_30_DAYS_POLICY = new LifecyclePolicyConfig(
ILM_30_DAYS_POLICY_NAME,
"/" + ILM_30_DAYS_POLICY_NAME + ".json"
);
public static final LifecyclePolicyConfig ILM_90_DAYS_POLICY = new LifecyclePolicyConfig(
ILM_90_DAYS_POLICY_NAME,
"/" + ILM_90_DAYS_POLICY_NAME + ".json"
);
public static final LifecyclePolicyConfig ILM_180_DAYS_POLICY = new LifecyclePolicyConfig(
ILM_180_DAYS_POLICY_NAME,
"/" + ILM_180_DAYS_POLICY_NAME + ".json"
);
public static final LifecyclePolicyConfig ILM_365_DAYS_POLICY = new LifecyclePolicyConfig(
ILM_365_DAYS_POLICY_NAME,
"/" + ILM_365_DAYS_POLICY_NAME + ".json"
);

//////////////////////////////////////////////////////////
// Logs components (for matching logs-*-* indices)
//////////////////////////////////////////////////////////
Expand Down Expand Up @@ -177,7 +207,16 @@ private void updateEnabledSetting(boolean newValue) {
@Override
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
if (stackTemplateEnabled) {
return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY, SYNTHETICS_ILM_POLICY);
return Arrays.asList(
LOGS_ILM_POLICY,
METRICS_ILM_POLICY,
SYNTHETICS_ILM_POLICY,
ILM_7_DAYS_POLICY,
ILM_30_DAYS_POLICY,
ILM_90_DAYS_POLICY,
ILM_180_DAYS_POLICY,
ILM_365_DAYS_POLICY
);
} else {
return Collections.emptyList();
}
Expand Down
17 changes: 13 additions & 4 deletions .../plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,15 @@
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.ilm.DeleteAction;
import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
import org.elasticsearch.xpack.core.ilm.LifecycleAction;
import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata;
import org.elasticsearch.xpack.core.ilm.LifecycleType;
import org.elasticsearch.xpack.core.ilm.OperationMode;
import org.elasticsearch.xpack.core.ilm.RolloverAction;
import org.elasticsearch.xpack.core.ilm.ShrinkAction;
import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
import org.junit.After;
Expand Down Expand Up @@ -91,6 +93,8 @@ public void createRegistryAndClient() {
(p) -> TimeseriesLifecycleType.INSTANCE
),
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse),
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse)
)
);
Expand Down Expand Up @@ -154,7 +158,12 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
anyOf(
equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME),
equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME),
equalTo(StackTemplateRegistry.SYNTHETICS_ILM_POLICY_NAME)
equalTo(StackTemplateRegistry.SYNTHETICS_ILM_POLICY_NAME),
equalTo(StackTemplateRegistry.ILM_7_DAYS_POLICY_NAME),
equalTo(StackTemplateRegistry.ILM_30_DAYS_POLICY_NAME),
equalTo(StackTemplateRegistry.ILM_90_DAYS_POLICY_NAME),
equalTo(StackTemplateRegistry.ILM_180_DAYS_POLICY_NAME),
equalTo(StackTemplateRegistry.ILM_365_DAYS_POLICY_NAME)
)
);
assertNotNull(listener);
Expand All @@ -173,7 +182,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {

ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(3)));
assertBusy(() -> assertThat(calledTimes.get(), equalTo(8)));
}

public void testPolicyAlreadyExists() {
Expand All @@ -185,7 +194,7 @@ public void testPolicyAlreadyExists() {
.stream()
.map(policyConfig -> policyConfig.load(xContentRegistry))
.collect(Collectors.toList());
assertThat(policies, hasSize(3));
assertThat(policies, hasSize(8));
policies.forEach(p -> policyMap.put(p.getName(), p));

client.setVerifier((action, request, listener) -> {
Expand Down Expand Up @@ -214,7 +223,7 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException {
.stream()
.map(policyConfig -> policyConfig.load(xContentRegistry))
.collect(Collectors.toList());
assertThat(policies, hasSize(3));
assertThat(policies, hasSize(8));
policies.forEach(p -> policyMap.put(p.getName(), p));

client.setVerifier((action, request, listener) -> {
Expand Down

0 comments on commit 692f694

Please sign in to comment.