[8.0] Add ILM policy for new stack monitoring data streams (#82498) (#…

…82892) Adds a templated ILM policy with a 3 day or 50 gb rollover and a 3 day retention period by default. The retention period checks the value set in xpack.monitoring.history.duration and uses that for the retention period if it is present. Depending on which default it uses, the _meta field is updated with the reasoning for picking the default retention value. Co-authored-by: Lee Hinman <[email protected]> Co-authored-by: Elastic Machine <[email protected]>
elastic · Jan 24, 2022 · 1d1af30 · 1d1af30
1 parent 7b93468
commit 1d1af30
Show file tree

Hide file tree

Showing 14 changed files with 590 additions and 9 deletions.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -581,7 +581,8 @@ protected Set<String> preserveILMPolicyIds() {
             "180-days-default",
             "365-days-default",
             ".fleet-actions-results-ilm-policy",
-            ".deprecation-indexing-ilm-policy"
+            ".deprecation-indexing-ilm-policy",
+            ".monitoring-8-ilm-policy"
         );
     }
 

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java
@@ -28,6 +28,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
@@ -40,9 +42,15 @@ public class LifecyclePolicyUtils {
     /**
      * Loads a built-in index lifecycle policy and returns its source.
      */
-    public static LifecyclePolicy loadPolicy(String name, String resource, NamedXContentRegistry xContentRegistry) {
+    public static LifecyclePolicy loadPolicy(
+        String name,
+        String resource,
+        Map<String, String> variables,
+        NamedXContentRegistry xContentRegistry
+    ) {
         try {
             BytesReference source = load(resource);
+            source = replaceVariables(source, variables);
             validate(source);
 
             try (
@@ -70,6 +78,21 @@ private static BytesReference load(String name) throws IOException {
         }
     }
 
+    private static BytesReference replaceVariables(BytesReference input, Map<String, String> variables) {
+        String template = input.utf8ToString();
+        for (Map.Entry<String, String> variable : variables.entrySet()) {
+            template = replaceVariable(template, variable.getKey(), variable.getValue());
+        }
+        return new BytesArray(template);
+    }
+
+    /**
+     * Replaces all occurrences of given variable with the value
+     */
+    public static String replaceVariable(String input, String variable, String value) {
+        return Pattern.compile("${" + variable + "}", Pattern.LITERAL).matcher(input).replaceAll(value);
+    }
+
     /**
      * Parses and validates that the source is not empty.
      */

diff --git a/...lugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java b/...lugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java
@@ -11,13 +11,17 @@
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils;
 
+import java.util.Collections;
+import java.util.Map;
+
 /**
  * Describes an index lifecycle policy to be loaded from a resource file for use with an {@link IndexTemplateRegistry}.
  */
 public class LifecyclePolicyConfig {
 
     private final String policyName;
     private final String fileName;
+    private final Map<String, String> templateVariables;
 
     /**
      * Describes a lifecycle policy definition to be loaded from a resource file.
@@ -27,8 +31,21 @@ public class LifecyclePolicyConfig {
      *                 extension if necessary.
      */
     public LifecyclePolicyConfig(String policyName, String fileName) {
+        this(policyName, fileName, Collections.emptyMap());
+    }
+
+    /**
+     * Describes a lifecycle policy definition to be loaded from a resource file.
+     *
+     * @param policyName The name that will be used for the policy.
+     * @param fileName The filename the policy definition should be loaded from. Literal, should include leading {@literal /} and
+     *                 extension if necessary.
+     * @param templateVariables A map containing values for template variables present in the resource file.
+     */
+    public LifecyclePolicyConfig(String policyName, String fileName, Map<String, String> templateVariables) {
         this.policyName = policyName;
         this.fileName = fileName;
+        this.templateVariables = templateVariables;
     }
 
     public String getPolicyName() {
@@ -40,6 +57,6 @@ public String getFileName() {
     }
 
     public LifecyclePolicy load(NamedXContentRegistry xContentRegistry) {
-        return LifecyclePolicyUtils.loadPolicy(policyName, fileName, xContentRegistry);
+        return LifecyclePolicyUtils.loadPolicy(policyName, fileName, templateVariables, xContentRegistry);
     }
 }
diff --git a/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json
@@ -2192,6 +2192,9 @@
           "path": "beat.elasticsearch.cluster.id"
         }
       }
+    },
+    "settings": {
+      "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}"
     }
   },
   "data_stream": {}

diff --git a/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json
@@ -3292,7 +3292,8 @@
       }
     },
     "settings": {
-      "index.mapping.total_fields.limit": 2000
+      "index.mapping.total_fields.limit": 2000,
+      "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}"
     }
   },
   "data_stream": {}

diff --git a/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json
@@ -644,7 +644,8 @@
       }
     },
     "settings": {
-      "index.mapping.total_fields.limit": 2000
+      "index.mapping.total_fields.limit": 2000,
+      "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}"
     }
   },
   "data_stream": {}

diff --git a/x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json
@@ -750,7 +750,8 @@
       }
     },
     "settings": {
-      "index.mapping.total_fields.limit": 2000
+      "index.mapping.total_fields.limit": 2000,
+      "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}"
     }
   },
   "data_stream": {}

diff --git a/x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json b/x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json
@@ -0,0 +1,31 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_primary_shard_size": "50gb",
+          "max_age": "3d"
+        }
+      }
+    },
+    "warm": {
+      "actions": {
+        "forcemerge": {
+          "max_num_segments": 1
+        }
+      }
+    },
+    "delete": {
+      "min_age": "${xpack.stack.monitoring.history.duration}",
+      "actions":{
+        "delete": {}
+      }
+    }
+  },
+  "_meta": {
+    "description": "Index lifecycle policy generated for [monitoring-*-8] data streams",
+    "defaults": {
+      "delete_min_age": "Using value of [${xpack.stack.monitoring.history.duration}] based on ${xpack.stack.monitoring.history.duration.reason}"
+    }
+  }
+}
diff --git a/...s/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/...s/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
@@ -57,6 +57,7 @@
 import org.elasticsearch.xpack.core.action.CreateDataStreamAction;
 import org.elasticsearch.xpack.core.action.DeleteDataStreamAction;
 import org.elasticsearch.xpack.core.ilm.DeleteAction;
+import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
@@ -313,6 +314,7 @@ protected void ensureClusterStateConsistency() throws IOException {
                 )
             );
             entries.add(new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new));
+            entries.add(new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new));
             entries.add(new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new));
             entries.add(
                 new NamedWriteableRegistry.Entry(

diff --git a/...nitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java b/...nitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java
@@ -17,9 +17,11 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
+import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -29,6 +31,8 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.monitoring.MonitoringField.HISTORY_DURATION;
+
 /**
  * Template registry for monitoring templates. Templates are loaded and installed shortly after cluster startup.
  *
@@ -58,6 +62,16 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry {
     private static final String TEMPLATE_VERSION_VARIABLE = "xpack.monitoring.template.version";
     private static final Map<String, String> ADDITIONAL_TEMPLATE_VARIABLES = Map.of(TEMPLATE_VERSION_VARIABLE, TEMPLATE_VERSION);
 
+    /**
+     * The stack monitoring ILM policy information. The template variables for the ILM policy are generated when the
+     * registry is created so that we can pick a default retention value that is sensitive to legacy monitoring settings.
+     */
+    public static final String MONITORING_POLICY_NAME = ".monitoring-8-ilm-policy";
+    private static final String MONITORING_POLICY_NAME_VARIABLE = "xpack.stack.monitoring.policy.name";
+    public static final String MONITORING_POLICY_DEFAULT_RETENTION = "3d";
+    private static final String MONITORING_POLICY_RETENTION_VARIABLE = "xpack.stack.monitoring.history.duration";
+    private static final String MONITORING_POLICY_RETENTION_REASON_VARIABLE = "xpack.stack.monitoring.history.duration.reason";
+
     /**
      * The stack monitoring template registry version. This is the version id for templates used by Metricbeat in version 8.x. Metricbeat
      * writes monitoring data in ECS format as of 8.0. These templates define the ECS schema as well as alias fields for the old monitoring
@@ -67,7 +81,12 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry {
     private static final String STACK_MONITORING_REGISTRY_VERSION_VARIABLE = "xpack.stack.monitoring.template.release.version";
     private static final String STACK_TEMPLATE_VERSION = "8";
     private static final String STACK_TEMPLATE_VERSION_VARIABLE = "xpack.stack.monitoring.template.version";
-    private static final Map<String, String> STACK_TEMPLATE_VARIABLES = Map.of(STACK_TEMPLATE_VERSION_VARIABLE, STACK_TEMPLATE_VERSION);
+    private static final Map<String, String> STACK_TEMPLATE_VARIABLES = Map.of(
+        STACK_TEMPLATE_VERSION_VARIABLE,
+        STACK_TEMPLATE_VERSION,
+        MONITORING_POLICY_NAME_VARIABLE,
+        MONITORING_POLICY_NAME
+    );
 
     public static final Setting<Boolean> MONITORING_TEMPLATES_ENABLED = Setting.boolSetting(
         "xpack.monitoring.templates.enabled",
@@ -207,6 +226,8 @@ public static IndexTemplateConfig getTemplateConfigForMonitoredSystem(MonitoredS
             .orElseThrow(() -> new IllegalArgumentException("Invalid system [" + system + "]"));
     }
 
+    private final List<LifecyclePolicyConfig> ilmPolicies;
+
     public MonitoringTemplateRegistry(
         Settings nodeSettings,
         ClusterService clusterService,
@@ -217,6 +238,27 @@ public MonitoringTemplateRegistry(
         super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
         this.clusterService = clusterService;
         this.monitoringTemplatesEnabled = MONITORING_TEMPLATES_ENABLED.get(nodeSettings);
+        this.ilmPolicies = loadPolicies(nodeSettings);
+    }
+
+    private List<LifecyclePolicyConfig> loadPolicies(Settings nodeSettings) {
+        Map<String, String> templateVars = new HashMap<>();
+        if (HISTORY_DURATION.exists(nodeSettings)) {
+            templateVars.put(MONITORING_POLICY_RETENTION_VARIABLE, HISTORY_DURATION.get(nodeSettings).getStringRep());
+            templateVars.put(
+                MONITORING_POLICY_RETENTION_REASON_VARIABLE,
+                "the value of the [" + HISTORY_DURATION.getKey() + "] setting at node startup"
+            );
+        } else {
+            templateVars.put(MONITORING_POLICY_RETENTION_VARIABLE, MONITORING_POLICY_DEFAULT_RETENTION);
+            templateVars.put(MONITORING_POLICY_RETENTION_REASON_VARIABLE, "the monitoring plugin default");
+        }
+        LifecyclePolicyConfig monitoringPolicy = new LifecyclePolicyConfig(
+            MONITORING_POLICY_NAME,
+            "/monitoring-mb-ilm-policy.json",
+            templateVars
+        );
+        return Collections.singletonList(monitoringPolicy);
     }
 
     @Override
@@ -266,6 +308,19 @@ protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
         }
     }
 
+    @Override
+    protected List<LifecyclePolicyConfig> getPolicyConfigs() {
+        if (monitoringTemplatesEnabled) {
+            return ilmPolicies;
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    List<LifecyclePolicy> getPolicies() {
+        return getPolicyConfigs().stream().map(p -> p.load(this.xContentRegistry)).collect(Collectors.toList());
+    }
+
     @Override
     protected String getOrigin() {
         return ClientHelper.MONITORING_ORIGIN;