Add ILM policy for new stack monitoring data streams

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java

@@ -28,6 +28,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
@@ -40,9 +42,15 @@ public class LifecyclePolicyUtils {
     /**
      * Loads a built-in index lifecycle policy and returns its source.
      */
-    public static LifecyclePolicy loadPolicy(String name, String resource, NamedXContentRegistry xContentRegistry) {
+    public static LifecyclePolicy loadPolicy(
+        String name,
+        String resource,
+        Map<String, String> variables,
+        NamedXContentRegistry xContentRegistry
+    ) {
         try {
             BytesReference source = load(resource);
+            source = replaceVariables(source, variables);
             validate(source);
 
             try (
@@ -70,6 +78,21 @@ private static BytesReference load(String name) throws IOException {
         }
     }
 
+    private static BytesReference replaceVariables(BytesReference input, Map<String, String> variables) {
+        String template = input.utf8ToString();
+        for (Map.Entry<String, String> variable : variables.entrySet()) {
+            template = replaceVariable(template, variable.getKey(), variable.getValue());
+        }
+        return new BytesArray(template);
+    }
+
+    /**
+     * Replaces all occurrences of given variable with the value
+     */
+    public static String replaceVariable(String input, String variable, String value) {
+        return Pattern.compile("${" + variable + "}", Pattern.LITERAL).matcher(input).replaceAll(value);
+    }
+
     /**
      * Parses and validates that the source is not empty.
      */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java

@@ -19,7 +19,9 @@
 import org.elasticsearch.xpack.core.ilm.ShrinkAction;
 import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
 
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Describes an index lifecycle policy to be loaded from a resource file for use with an {@link IndexTemplateRegistry}.
@@ -42,6 +44,7 @@ public class LifecyclePolicyConfig {
 
     private final String policyName;
     private final String fileName;
+    private final Map<String, String> templateVariables;
 
     /**
      * Describes a lifecycle policy definition to be loaded from a resource file.
@@ -51,8 +54,21 @@ public class LifecyclePolicyConfig {
      *                 extension if necessary.
      */
     public LifecyclePolicyConfig(String policyName, String fileName) {
+        this(policyName, fileName, Collections.emptyMap());
+    }
+
+    /**
+     * Describes a lifecycle policy definition to be loaded from a resource file.
+     *
+     * @param policyName The name that will be used for the policy.
+     * @param fileName The filename the policy definition should be loaded from. Literal, should include leading {@literal /} and
+     *                 extension if necessary.
+     * @param templateVariables A map containing values for template variables present in the resource file.
+     */
+    public LifecyclePolicyConfig(String policyName, String fileName, Map<String, String> templateVariables) {
         this.policyName = policyName;
         this.fileName = fileName;
+        this.templateVariables = templateVariables;
     }
 
     public String getPolicyName() {
@@ -64,6 +80,6 @@ public String getFileName() {
     }
 
     public LifecyclePolicy load(NamedXContentRegistry xContentRegistry) {
-        return LifecyclePolicyUtils.loadPolicy(policyName, fileName, xContentRegistry);
+        return LifecyclePolicyUtils.loadPolicy(policyName, fileName, templateVariables, xContentRegistry);
     }
 }

x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json

@@ -2192,6 +2192,9 @@
           "path": "beat.elasticsearch.cluster.id"
         }
       }
+    },
+    "settings": {
+      "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}"
     }
   },
   "data_stream": {}

x-pack/plugin/core/src/main/resources/monitoring-es-mb.json

@@ -3292,7 +3292,8 @@
       }
     },
     "settings": {
-      "index.mapping.total_fields.limit": 2000
+      "index.mapping.total_fields.limit": 2000,
+      "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}"
     }
   },
   "data_stream": {}

x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json

@@ -644,7 +644,8 @@
       }
     },
     "settings": {
-      "index.mapping.total_fields.limit": 2000
+      "index.mapping.total_fields.limit": 2000,
+      "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}"
     }
   },
   "data_stream": {}

x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json

@@ -750,7 +750,8 @@
       }
     },
     "settings": {
-      "index.mapping.total_fields.limit": 2000
+      "index.mapping.total_fields.limit": 2000,
+      "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}"
     }
   },
   "data_stream": {}

x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json

@@ -0,0 +1,31 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_primary_shard_size": "50gb",
+          "max_age": "3d"
+        }
+      }
+    },
+    "warm": {
+      "actions": {
+        "forcemerge": {
+          "max_num_segments": 1
+        }
+      }
+    },
+    "delete": {
+      "min_age": "${xpack.stack.monitoring.history.duration}",
+      "actions":{
+        "delete": {}
+      }
+    }
+  },
+  "_meta": {
+    "description": "Index lifecycle policy generated for [monitoring-*-8] data streams",
+    "defaults": {
+      "delete_min_age": "Using value of [${xpack.stack.monitoring.history.duration}] based on ${xpack.stack.monitoring.history.duration.reason}"
+    }
+  }
+}

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java

@@ -18,9 +18,11 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
+import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -30,6 +32,8 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.monitoring.MonitoringField.HISTORY_DURATION;
+
 /**
  * Template registry for monitoring templates. Templates are loaded and installed shortly after cluster startup.
  *
@@ -59,6 +63,16 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry {
     private static final String TEMPLATE_VERSION_VARIABLE = "xpack.monitoring.template.version";
     private static final Map<String, String> ADDITIONAL_TEMPLATE_VARIABLES = Map.of(TEMPLATE_VERSION_VARIABLE, TEMPLATE_VERSION);
 
+    /**
+     * The stack monitoring ILM policy information. The template variables for the ILM policy are generated when the
+     * registry is created so that we can pick a default retention value that is sensitive to legacy monitoring settings.
+     */
+    public static final String MONITORING_POLICY_NAME = ".monitoring-8-ilm-policy";
+    private static final String MONITORING_POLICY_NAME_VARIABLE = "xpack.stack.monitoring.policy.name";
+    public static final String MONITORING_POLICY_DEFAULT_RETENTION = "3d";
+    private static final String MONITORING_POLICY_RETENTION_VARIABLE = "xpack.stack.monitoring.history.duration";
+    private static final String MONITORING_POLICY_RETENTION_REASON_VARIABLE = "xpack.stack.monitoring.history.duration.reason";
+
     /**
      * The stack monitoring template registry version. This is the version id for templates used by Metricbeat in version 8.x. Metricbeat
      * writes monitoring data in ECS format as of 8.0. These templates define the ECS schema as well as alias fields for the old monitoring
@@ -68,7 +82,12 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry {
     private static final String STACK_MONITORING_REGISTRY_VERSION_VARIABLE = "xpack.stack.monitoring.template.release.version";
     private static final String STACK_TEMPLATE_VERSION = "8";
     private static final String STACK_TEMPLATE_VERSION_VARIABLE = "xpack.stack.monitoring.template.version";
-    private static final Map<String, String> STACK_TEMPLATE_VARIABLES = Map.of(STACK_TEMPLATE_VERSION_VARIABLE, STACK_TEMPLATE_VERSION);
+    private static final Map<String, String> STACK_TEMPLATE_VARIABLES = Map.of(
+        STACK_TEMPLATE_VERSION_VARIABLE,
+        STACK_TEMPLATE_VERSION,
+        MONITORING_POLICY_NAME_VARIABLE,
+        MONITORING_POLICY_NAME
+    );
 
     public static final Setting<Boolean> MONITORING_TEMPLATES_ENABLED = Setting.boolSetting(
         "xpack.monitoring.templates.enabled",
@@ -208,6 +227,8 @@ public static IndexTemplateConfig getTemplateConfigForMonitoredSystem(MonitoredS
             .orElseThrow(() -> new IllegalArgumentException("Invalid system [" + system + "]"));
     }
 
+    private final List<LifecyclePolicy> ilmPolicies;
+
     public MonitoringTemplateRegistry(
         Settings nodeSettings,
         ClusterService clusterService,
@@ -218,6 +239,24 @@ public MonitoringTemplateRegistry(
         super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
         this.clusterService = clusterService;
         this.monitoringTemplatesEnabled = MONITORING_TEMPLATES_ENABLED.get(nodeSettings);
+        this.ilmPolicies = loadPolicies(nodeSettings);
+    }
+
+    private List<LifecyclePolicy> loadPolicies(Settings nodeSettings) {
+        Map<String, String> templateVars = new HashMap<>();
+        if (HISTORY_DURATION.exists(nodeSettings)) {
+            templateVars.put(MONITORING_POLICY_RETENTION_VARIABLE, HISTORY_DURATION.get(nodeSettings).getStringRep());
+            templateVars.put(
+                MONITORING_POLICY_RETENTION_REASON_VARIABLE,
+                "the value of the [" + HISTORY_DURATION.getKey() + "] setting at node startup"
+            );
+        } else {
+            templateVars.put(MONITORING_POLICY_RETENTION_VARIABLE, MONITORING_POLICY_DEFAULT_RETENTION);
+            templateVars.put(MONITORING_POLICY_RETENTION_REASON_VARIABLE, "the monitoring plugin default");
+        }
+        LifecyclePolicy monitoringPolicy = new LifecyclePolicyConfig(MONITORING_POLICY_NAME, "/monitoring-mb-ilm-policy.json", templateVars)
+            .load(LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY);
+        return Collections.singletonList(monitoringPolicy);
     }
 
     @Override
@@ -265,6 +304,15 @@ protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
         return monitoringTemplatesEnabled ? COMPOSABLE_INDEX_TEMPLATE_CONFIGS : Map.of();
     }
 
+    @Override
+    protected List<LifecyclePolicy> getPolicyConfigs() {
+        if (monitoringTemplatesEnabled) {
+            return ilmPolicies;
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
     @Override
     protected String getOrigin() {
         return ClientHelper.MONITORING_ORIGIN;