diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 07d54d3dc4b6e..9371a2efb77a1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -581,7 +581,8 @@ protected Set preserveILMPolicyIds() { "180-days-default", "365-days-default", ".fleet-actions-results-ilm-policy", - ".deprecation-indexing-ilm-policy" + ".deprecation-indexing-ilm-policy", + ".monitoring-8-ilm-policy" ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java index fe6cb5f6f8e9c..6534079aab675 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java @@ -28,6 +28,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -40,9 +42,15 @@ public class LifecyclePolicyUtils { /** * Loads a built-in index lifecycle policy and returns its source. */ - public static LifecyclePolicy loadPolicy(String name, String resource, NamedXContentRegistry xContentRegistry) { + public static LifecyclePolicy loadPolicy( + String name, + String resource, + Map variables, + NamedXContentRegistry xContentRegistry + ) { try { BytesReference source = load(resource); + source = replaceVariables(source, variables); validate(source); try ( @@ -70,6 +78,21 @@ private static BytesReference load(String name) throws IOException { } } + private static BytesReference replaceVariables(BytesReference input, Map variables) { + String template = input.utf8ToString(); + for (Map.Entry variable : variables.entrySet()) { + template = replaceVariable(template, variable.getKey(), variable.getValue()); + } + return new BytesArray(template); + } + + /** + * Replaces all occurrences of given variable with the value + */ + public static String replaceVariable(String input, String variable, String value) { + return Pattern.compile("${" + variable + "}", Pattern.LITERAL).matcher(input).replaceAll(value); + } + /** * Parses and validates that the source is not empty. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java index bc0e1f5ec7c7e..b7d69bc160d2a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java @@ -19,7 +19,9 @@ import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; +import java.util.Collections; import java.util.List; +import java.util.Map; /** * Describes an index lifecycle policy to be loaded from a resource file for use with an {@link IndexTemplateRegistry}. @@ -42,6 +44,7 @@ public class LifecyclePolicyConfig { private final String policyName; private final String fileName; + private final Map templateVariables; /** * Describes a lifecycle policy definition to be loaded from a resource file. @@ -51,8 +54,21 @@ public class LifecyclePolicyConfig { * extension if necessary. */ public LifecyclePolicyConfig(String policyName, String fileName) { + this(policyName, fileName, Collections.emptyMap()); + } + + /** + * Describes a lifecycle policy definition to be loaded from a resource file. + * + * @param policyName The name that will be used for the policy. + * @param fileName The filename the policy definition should be loaded from. Literal, should include leading {@literal /} and + * extension if necessary. + * @param templateVariables A map containing values for template variables present in the resource file. + */ + public LifecyclePolicyConfig(String policyName, String fileName, Map templateVariables) { this.policyName = policyName; this.fileName = fileName; + this.templateVariables = templateVariables; } public String getPolicyName() { @@ -64,6 +80,6 @@ public String getFileName() { } public LifecyclePolicy load(NamedXContentRegistry xContentRegistry) { - return LifecyclePolicyUtils.loadPolicy(policyName, fileName, xContentRegistry); + return LifecyclePolicyUtils.loadPolicy(policyName, fileName, templateVariables, xContentRegistry); } } diff --git a/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json index 001c6fdfaf3a5..787f78304ea0e 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json @@ -2192,6 +2192,9 @@ "path": "beat.elasticsearch.cluster.id" } } + }, + "settings": { + "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}" } }, "data_stream": {} diff --git a/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json index bc548d46066c0..501250f48be99 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json @@ -3292,7 +3292,8 @@ } }, "settings": { - "index.mapping.total_fields.limit": 2000 + "index.mapping.total_fields.limit": 2000, + "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}" } }, "data_stream": {} diff --git a/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json index 3a47a16f08b04..e155f74ae0486 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json @@ -644,7 +644,8 @@ } }, "settings": { - "index.mapping.total_fields.limit": 2000 + "index.mapping.total_fields.limit": 2000, + "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}" } }, "data_stream": {} diff --git a/x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json index 55ce05bbc913c..fff21e76dd0b2 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json @@ -750,7 +750,8 @@ } }, "settings": { - "index.mapping.total_fields.limit": 2000 + "index.mapping.total_fields.limit": 2000, + "index.lifecycle.name": "${xpack.stack.monitoring.policy.name}" } }, "data_stream": {} diff --git a/x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json b/x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json new file mode 100644 index 0000000000000..8ef931aa2f12f --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json @@ -0,0 +1,31 @@ +{ + "phases": { + "hot": { + "actions": { + "rollover": { + "max_primary_shard_size": "50gb", + "max_age": "3d" + } + } + }, + "warm": { + "actions": { + "forcemerge": { + "max_num_segments": 1 + } + } + }, + "delete": { + "min_age": "${xpack.stack.monitoring.history.duration}", + "actions":{ + "delete": {} + } + } + }, + "_meta": { + "description": "Index lifecycle policy generated for [monitoring-*-8] data streams", + "defaults": { + "delete_min_age": "Using value of [${xpack.stack.monitoring.history.duration}] based on ${xpack.stack.monitoring.history.duration.reason}" + } + } +} diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 9dce0e1c41a05..11e0c29c84898 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -57,6 +57,7 @@ import org.elasticsearch.xpack.core.action.CreateDataStreamAction; import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.ForceMergeAction; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; @@ -313,6 +314,7 @@ protected void ensureClusterStateConsistency() throws IOException { ) ); entries.add(new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::readFrom)); + entries.add(new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new)); entries.add(new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new)); entries.add( new NamedWriteableRegistry.Entry( diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java index 4f4cae3a28f62..fe54eac3bebe2 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @@ -18,9 +18,11 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.monitoring.MonitoredSystem; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; import java.util.Arrays; import java.util.Collections; @@ -30,6 +32,8 @@ import java.util.Optional; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.monitoring.MonitoringField.HISTORY_DURATION; + /** * Template registry for monitoring templates. Templates are loaded and installed shortly after cluster startup. * @@ -59,6 +63,16 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry { private static final String TEMPLATE_VERSION_VARIABLE = "xpack.monitoring.template.version"; private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of(TEMPLATE_VERSION_VARIABLE, TEMPLATE_VERSION); + /** + * The stack monitoring ILM policy information. The template variables for the ILM policy are generated when the + * registry is created so that we can pick a default retention value that is sensitive to legacy monitoring settings. + */ + public static final String MONITORING_POLICY_NAME = ".monitoring-8-ilm-policy"; + private static final String MONITORING_POLICY_NAME_VARIABLE = "xpack.stack.monitoring.policy.name"; + public static final String MONITORING_POLICY_DEFAULT_RETENTION = "3d"; + private static final String MONITORING_POLICY_RETENTION_VARIABLE = "xpack.stack.monitoring.history.duration"; + private static final String MONITORING_POLICY_RETENTION_REASON_VARIABLE = "xpack.stack.monitoring.history.duration.reason"; + /** * The stack monitoring template registry version. This is the version id for templates used by Metricbeat in version 8.x. Metricbeat * writes monitoring data in ECS format as of 8.0. These templates define the ECS schema as well as alias fields for the old monitoring @@ -68,7 +82,12 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry { private static final String STACK_MONITORING_REGISTRY_VERSION_VARIABLE = "xpack.stack.monitoring.template.release.version"; private static final String STACK_TEMPLATE_VERSION = "8"; private static final String STACK_TEMPLATE_VERSION_VARIABLE = "xpack.stack.monitoring.template.version"; - private static final Map STACK_TEMPLATE_VARIABLES = Map.of(STACK_TEMPLATE_VERSION_VARIABLE, STACK_TEMPLATE_VERSION); + private static final Map STACK_TEMPLATE_VARIABLES = Map.of( + STACK_TEMPLATE_VERSION_VARIABLE, + STACK_TEMPLATE_VERSION, + MONITORING_POLICY_NAME_VARIABLE, + MONITORING_POLICY_NAME + ); public static final Setting MONITORING_TEMPLATES_ENABLED = Setting.boolSetting( "xpack.monitoring.templates.enabled", @@ -208,6 +227,8 @@ public static IndexTemplateConfig getTemplateConfigForMonitoredSystem(MonitoredS .orElseThrow(() -> new IllegalArgumentException("Invalid system [" + system + "]")); } + private final List ilmPolicies; + public MonitoringTemplateRegistry( Settings nodeSettings, ClusterService clusterService, @@ -218,6 +239,24 @@ public MonitoringTemplateRegistry( super(nodeSettings, clusterService, threadPool, client, xContentRegistry); this.clusterService = clusterService; this.monitoringTemplatesEnabled = MONITORING_TEMPLATES_ENABLED.get(nodeSettings); + this.ilmPolicies = loadPolicies(nodeSettings); + } + + private List loadPolicies(Settings nodeSettings) { + Map templateVars = new HashMap<>(); + if (HISTORY_DURATION.exists(nodeSettings)) { + templateVars.put(MONITORING_POLICY_RETENTION_VARIABLE, HISTORY_DURATION.get(nodeSettings).getStringRep()); + templateVars.put( + MONITORING_POLICY_RETENTION_REASON_VARIABLE, + "the value of the [" + HISTORY_DURATION.getKey() + "] setting at node startup" + ); + } else { + templateVars.put(MONITORING_POLICY_RETENTION_VARIABLE, MONITORING_POLICY_DEFAULT_RETENTION); + templateVars.put(MONITORING_POLICY_RETENTION_REASON_VARIABLE, "the monitoring plugin default"); + } + LifecyclePolicy monitoringPolicy = new LifecyclePolicyConfig(MONITORING_POLICY_NAME, "/monitoring-mb-ilm-policy.json", templateVars) + .load(LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY); + return Collections.singletonList(monitoringPolicy); } @Override @@ -265,6 +304,15 @@ protected Map getComposableTemplateConfigs() { return monitoringTemplatesEnabled ? COMPOSABLE_INDEX_TEMPLATE_CONFIGS : Map.of(); } + @Override + protected List getPolicyConfigs() { + if (monitoringTemplatesEnabled) { + return ilmPolicies; + } else { + return Collections.emptyList(); + } + } + @Override protected String getOrigin() { return ClientHelper.MONITORING_ORIGIN; diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistryTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistryTests.java new file mode 100644 index 0000000000000..61b8806a0dec6 --- /dev/null +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistryTests.java @@ -0,0 +1,431 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.monitoring; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.core.monitoring.MonitoringField.HISTORY_DURATION; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class MonitoringTemplateRegistryTests extends ESTestCase { + private MonitoringTemplateRegistry registry; + private ClusterService clusterService; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + registry = new MonitoringTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + client.setVerifier((a, r, l) -> { + fail("if the master is missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.singletonMap(MonitoringTemplateRegistry.ES_INDEX_TEMPLATE_NAME, null), + nodes + ); + registry.clusterChanged(event); + } + + public void testDisabledDoesNotAddTemplates() { + Settings settings = Settings.builder().put(MonitoringTemplateRegistry.MONITORING_TEMPLATES_ENABLED.getKey(), false).build(); + MonitoringTemplateRegistry disabledRegistry = new MonitoringTemplateRegistry( + settings, + clusterService, + threadPool, + client, + NamedXContentRegistry.EMPTY + ); + assertThat(disabledRegistry.getLegacyTemplateConfigs(), is(empty())); + assertThat(disabledRegistry.getComposableTemplateConfigs(), anEmptyMap()); + assertThat(disabledRegistry.getPolicyConfigs(), hasSize(0)); + } + + public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComposableTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComposableTemplateConfigs().size()))); + + calledTimes.set(0); + + // attempting to register the event multiple times as a race condition can yield this test flaky, namely: + // when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains + // might've not yet been updated to reflect that the first template registration was complete, so a second template registration + // will not be issued anymore, leaving calledTimes to 0 + assertBusy(() -> { + // now delete one template from the cluster state and lets retry + ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), nodes); + registry.clusterChanged(newEvent); + assertThat(calledTimes.get(), greaterThan(1)); + }); + } + + public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + boolean historyDurationPresent = randomBoolean(); + final String expectedDeleteMinAge; + if (historyDurationPresent) { + expectedDeleteMinAge = randomTimeValue(4, 10, "d"); + } else { + expectedDeleteMinAge = MonitoringTemplateRegistry.MONITORING_POLICY_DEFAULT_RETENTION; + } + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutLifecycleAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutLifecycleAction.class)); + assertThat(request, instanceOf(PutLifecycleAction.Request.class)); + final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request; + assertThat(putRequest.getPolicy().getName(), equalTo(MonitoringTemplateRegistry.MONITORING_POLICY_NAME)); + if (putRequest.getPolicy().getName().equals(MonitoringTemplateRegistry.MONITORING_POLICY_NAME)) { + Phase delete = putRequest.getPolicy().getPhases().get("delete"); + assertThat(delete.getMinimumAge().getStringRep(), equalTo(expectedDeleteMinAge)); + } + assertNotNull(listener); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return new MonitoringTemplateRegistryTests.TestPutIndexTemplateResponse(true); + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + return AcknowledgedResponse.TRUE; + } else { + fail("client called with unexpected request: " + request.toString()); + return null; + } + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); + if (historyDurationPresent) { + Settings testSettings = Settings.builder().put(HISTORY_DURATION.getKey(), expectedDeleteMinAge).build(); + MonitoringTemplateRegistry testRegistry = new MonitoringTemplateRegistry( + testSettings, + clusterService, + threadPool, + client, + NamedXContentRegistry.EMPTY + ); + testRegistry.clusterChanged(event); + } else { + registry.clusterChanged(event); + } + assertBusy(() -> assertThat(calledTimes.get(), equalTo(1))); + + // Make sure we logged a deprecation for using the history setting + if (historyDurationPresent) { + assertSettingDeprecationsAndWarnings(new Setting[] { HISTORY_DURATION }); + } + } + + public void testPolicyAlreadyExists() { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + List policies = registry.getPolicyConfigs(); + assertThat(policies, hasSize(1)); + policies.forEach(p -> policyMap.put(p.getName(), p)); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should not be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + + public void testPolicyAlreadyExistsButDiffers() throws IOException { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}"; + List policies = registry.getPolicyConfigs(); + assertThat(policies, hasSize(1)); + policies.forEach(p -> policyMap.put(p.getName(), p)); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should not be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser( + XContentParserConfiguration.EMPTY.withRegistry( + new NamedXContentRegistry( + List.of( + new NamedXContentRegistry.Entry( + LifecycleAction.class, + new ParseField(DeleteAction.NAME), + DeleteAction::parse + ) + ) + ) + ), + policyStr + ) + ) { + LifecyclePolicy different = LifecyclePolicy.parse(parser, policies.get(0).getName()); + policyMap.put(policies.get(0).getName(), different); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + } + + public void testThatVersionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent( + Collections.singletonMap( + MonitoringTemplateRegistry.ES_STACK_INDEX_TEMPLATE_NAME, + MonitoringTemplateRegistry.STACK_MONITORING_REGISTRY_VERSION - 1 + ), + nodes + ); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyComposableTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComposableTemplateConfigs().size()))); + } + + /** + * A client that delegates to a verifying function for action/request/listener + */ + public static class VerifyingClient extends NoOpClient { + + private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { + fail("verifier not set"); + return null; + }; + + VerifyingClient(ThreadPool threadPool) { + super(threadPool); + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + this.verifier = verifier; + return this; + } + } + + private ActionResponse verifyComposableTemplateInstalled( + AtomicInteger calledTimes, + ActionType action, + ActionRequest request, + ActionListener listener + ) { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return new TestPutIndexTemplateResponse(true); + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutComposableIndexTemplateAction.class)); + assertThat(request, instanceOf(PutComposableIndexTemplateAction.Request.class)); + final PutComposableIndexTemplateAction.Request putRequest = ((PutComposableIndexTemplateAction.Request) request); + assertThat(putRequest.indexTemplate().version(), equalTo((long) MonitoringTemplateRegistry.STACK_MONITORING_REGISTRY_VERSION)); + assertNotNull(listener); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplates, DiscoveryNodes nodes) { + return createClusterChangedEvent(existingTemplates, Collections.emptyMap(), nodes); + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + ClusterState cs = createClusterState(Settings.EMPTY, existingTemplates, existingPolicies, nodes); + ClusterChangedEvent realEvent = new ClusterChangedEvent( + "created-from-test", + cs, + ClusterState.builder(new ClusterName("test")).build() + ); + ClusterChangedEvent event = spy(realEvent); + when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); + + return event; + } + + private ClusterState createClusterState( + Settings nodeSettings, + Map existingComposableTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + Map composableTemplates = new HashMap<>(); + for (Map.Entry template : existingComposableTemplates.entrySet()) { + ComposableIndexTemplate mockTemplate = mock(ComposableIndexTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + composableTemplates.put(template.getKey(), mockTemplate); + } + + Map existingILMMeta = existingPolicies.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new LifecyclePolicyMetadata(e.getValue(), Collections.emptyMap(), 1, 1))); + IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(existingILMMeta, OperationMode.RUNNING); + + return ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .indexTemplates(composableTemplates) + .transientSettings(nodeSettings) + .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) + .build() + ) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodes) + .build(); + } + + private static class TestPutIndexTemplateResponse extends AcknowledgedResponse { + TestPutIndexTemplateResponse(boolean acknowledged) { + super(acknowledged); + } + } +} diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index 080c8dbde2785..ee65c39f1ec59 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -18,6 +18,7 @@ dependencies { api project(path: ':modules:transport-netty4') api project(path: ':plugins:transport-nio') + testImplementation project(path: xpackModule('ilm')) testImplementation project(path: xpackModule('monitoring')) testImplementation project(path: xpackModule('spatial')) testImplementation project(path: xpackModule('vectors')) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java index 2ab5974db9e12..6e10a850fa508 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageResponse; import org.elasticsearch.xpack.core.ssl.SSLService; +import org.elasticsearch.xpack.ilm.IndexLifecycle; import org.elasticsearch.xpack.monitoring.Monitoring; import java.nio.file.Path; @@ -74,6 +75,12 @@ protected List infoActions() { public LocalStateSecurity(final Settings settings, final Path configPath) throws Exception { super(settings, configPath); LocalStateSecurity thisVar = this; + plugins.add(new IndexLifecycle(settings) { + @Override + protected XPackLicenseState getLicenseState() { + return thisVar.getLicenseState(); + } + }); plugins.add(new Monitoring(settings) { @Override protected SSLService getSslService() { diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java index 52c8be8814128..f3d41c3d96e44 100644 --- a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java +++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java @@ -182,7 +182,7 @@ public void testPolicyAlreadyExists() { // Ignore this, it's verified in another test return AcknowledgedResponse.TRUE; } else if (action instanceof PutLifecycleAction) { - fail("if the policy already exists it should be re-put"); + fail("if the policy already exists it should not be re-put"); } else { fail("client called with unexpected request: " + request.toString()); } @@ -208,7 +208,7 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException { // Ignore this, it's verified in another test return AcknowledgedResponse.TRUE; } else if (action instanceof PutLifecycleAction) { - fail("if the policy already exists it should be re-put"); + fail("if the policy already exists it should not be re-put"); } else { fail("client called with unexpected request: " + request.toString()); }