From b9f08145face018121ae293871bfef32271b5957 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 23 Feb 2021 11:41:29 +0000 Subject: [PATCH] Migrate watcher to system indices infrastructure (#67588) (#69328) Backport of #67588. Part of #61656. Migrate the `.watches` and `.triggered_watches` system indices to use the auto-create infrastructure. The watcher history indices are left alone. As part of this work, a `SystemIndexDescriptor` now inspects its mappings to determine whether it has any dynamic mappings. This influences how strict Elasticsearch is with enforcing the descriptor's mappings, since ES cannot know in advanced what all the mappings will be. This PR also fixes the `SystemIndexManager` so that (1) it doesn't fall over when attempting to inspect the state of an index that hasn't been created yet, and (2) does update mappings if there's no version field in the mapping metadata. --- .../put/TransportPutMappingAction.java | 2 +- .../indices/SystemIndexDescriptor.java | 42 +++ .../indices/SystemIndexManager.java | 34 ++- .../indices/SystemIndexDescriptorTests.java | 39 +++ .../indices/SystemIndexManagerTests.java | 13 + .../WatcherIndexTemplateRegistryField.java | 11 +- .../src/main/resources/triggered-watches.json | 46 --- .../core/src/main/resources/watches.json | 68 ----- .../AbstractWatcherIntegrationTestCase.java | 8 - .../elasticsearch/xpack/watcher/Watcher.java | 264 +++++++++++++----- .../support/WatcherIndexTemplateRegistry.java | 61 +--- .../watcher/WatcherLifeCycleServiceTests.java | 26 +- .../xpack/watcher/WatcherPluginTests.java | 28 -- .../WatcherIndexTemplateRegistryTests.java | 20 +- .../test/rest/XPackRestTestConstants.java | 4 +- 15 files changed, 336 insertions(+), 330 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/resources/triggered-watches.json delete mode 100644 x-pack/plugin/core/src/main/resources/watches.json diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 9c6940373cc8a..531c1810b2ede 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -150,7 +150,7 @@ static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] for (Index index : concreteIndices) { final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName()); - if (descriptor != null && descriptor.isAutomaticallyManaged()) { + if (descriptor != null && descriptor.isAutomaticallyManaged() && descriptor.hasDynamicMappings() == false) { final String descriptorMappings = descriptor.getMappings(); // Technically we could trip over a difference in whitespace here, but then again nobody should be trying to manually // update a descriptor's mappings. diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 6702a3e7a7605..c99ebe0fb8308 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -18,12 +18,15 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.MapperService; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; /** @@ -75,6 +78,9 @@ public class SystemIndexDescriptor { /** The minimum cluster node version required for this descriptor, or null if there is no restriction */ private final Version minimumNodeVersion; + /** Whether there are dynamic fields in this descriptor's mappings */ + private final boolean hasDynamicMappings; + /** * Creates a descriptor for system indices matching the supplied pattern. These indices will not be managed * by Elasticsearch internally. @@ -176,8 +182,12 @@ public SystemIndexDescriptor(String indexPattern, String description) { this.origin = origin; this.indexType = indexType; this.minimumNodeVersion = minimumNodeVersion; + + this.hasDynamicMappings = this.mappings != null + && findDynamicMapping(XContentHelper.convertToMap(JsonXContent.jsonXContent, mappings, false)); } + /** * @return The pattern of index names that this descriptor will be used for. */ @@ -266,6 +276,10 @@ public String getIndexType() { return indexType; } + public boolean hasDynamicMappings() { + return this.hasDynamicMappings; + } + /** * Checks that this descriptor can be used within this cluster e.g. the cluster supports all required * features, by comparing the supplied minimum node version to this descriptor's minimum version. @@ -436,4 +450,32 @@ private static String patternToRegex(String input) { output = output.replaceAll("\\*", ".*"); return output; } + + /** + * Recursively searches for dynamic: true in the supplies mappings + * @param map a parsed fragment of an index's mappings + * @return whether the fragment contains a dynamic mapping + */ + @SuppressWarnings("unchecked") + static boolean findDynamicMapping(Map map) { + if (map == null) { + return false; + } + + for (Map.Entry entry : map.entrySet()) { + final String key = entry.getKey(); + final Object value = entry.getValue(); + if (key.equals("dynamic") && (value instanceof Boolean) && ((Boolean) value)) { + return true; + } + + if (value instanceof Map) { + if (findDynamicMapping((Map) value)) { + return true; + } + } + } + + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java index f54e69a6f8aac..c2141491c7985 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java @@ -8,13 +8,6 @@ package org.elasticsearch.indices; -import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -39,6 +32,13 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.gateway.GatewayService; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING; + /** * This class ensures that all system indices have up-to-date mappings, provided * those indices can be automatically managed. Only some system indices are managed @@ -52,6 +52,11 @@ public class SystemIndexManager implements ClusterStateListener { private final Client client; private final AtomicBoolean isUpgradeInProgress; + /** + * Creates a new manager + * @param systemIndices the indices to manage + * @param client used to update the cluster + */ public SystemIndexManager(SystemIndices systemIndices, Client client) { this.systemIndices = systemIndices; this.client = client; @@ -138,6 +143,11 @@ UpgradeStatus getUpgradeStatus(ClusterState clusterState, SystemIndexDescriptor // The messages below will be logged on every cluster state update, which is why even in the index closed / red // cases, the log levels are DEBUG. + if (indexState == null) { + logger.debug("Index {} does not exist yet", indexDescription); + return UpgradeStatus.UP_TO_DATE; + } + if (indexState.indexState == IndexMetadata.State.CLOSE) { logger.debug("Index {} is closed. This is likely to prevent some features from functioning correctly", indexDescription); return UpgradeStatus.CLOSED; @@ -199,10 +209,16 @@ public void onFailure(Exception e) { /** * Derives a summary of the current state of a system index, relative to the given cluster state. + * @param state the cluster state from which to derive the index state + * @param descriptor the system index to check + * @return a summary of the index state, or null if the index doesn't exist */ State calculateIndexState(ClusterState state, SystemIndexDescriptor descriptor) { final IndexMetadata indexMetadata = state.metadata().index(descriptor.getPrimaryIndex()); - assert indexMetadata != null; + + if (indexMetadata == null) { + return null; + } final boolean isIndexUpToDate = INDEX_FORMAT_SETTING.get(indexMetadata.getSettings()) == descriptor.getIndexFormat(); @@ -256,6 +272,8 @@ private Version readMappingVersion(SystemIndexDescriptor descriptor, MappingMeta final String versionString = (String) meta.get(descriptor.getVersionMetaKey()); if (versionString == null) { logger.warn("No value found in mappings for [_meta.{}]", descriptor.getVersionMetaKey()); + // If we called `Version.fromString(null)`, it would return `Version.CURRENT` and we wouldn't update the mappings + return Version.V_EMPTY; } return Version.fromString(versionString); } catch (ElasticsearchParseException e) { diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java index 5778c7f4bc560..ce8fbdfbfb05b 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -8,13 +8,21 @@ package org.elasticsearch.indices; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; +import java.util.Map; + +import static org.elasticsearch.indices.SystemIndexDescriptor.findDynamicMapping; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class SystemIndexDescriptorTests extends ESTestCase { + /** + * Tests the various validation rules that are applied when creating a new system index descriptor. + */ public void testValidation() { { Exception ex = expectThrows(NullPointerException.class, @@ -75,4 +83,35 @@ public void testValidation() { ); } } + + /** + * Check that a system index descriptor correctly identifies the presence of a dynamic mapping when once is present. + */ + public void testFindDynamicMappingsWithDynamicMapping() { + String json = "{" + + " \"foo\": {" + + " \"bar\": {" + + " \"dynamic\": false" + + " }," + + " \"baz\": {" + + " \"dynamic\": true" + + " }" + + " }" + + "}"; + + final Map mappings = XContentHelper.convertToMap(JsonXContent.jsonXContent, json, false); + + assertThat(findDynamicMapping(mappings), equalTo(true)); + } + + /** + * Check that a system index descriptor correctly identifies the absence of a dynamic mapping when none are present. + */ + public void testFindDynamicMappingsWithoutDynamicMapping() { + String json = "{ \"foo\": { \"bar\": { \"dynamic\": false } } }"; + + final Map mappings = XContentHelper.convertToMap(JsonXContent.jsonXContent, json, false); + + assertThat(findDynamicMapping(mappings), equalTo(false)); + } } diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java index 87be259414715..621038931d76d 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java @@ -222,6 +222,19 @@ public void testManagerProcessesIndicesWithOutdatedMappings() { assertThat(manager.getUpgradeStatus(clusterStateBuilder.build(), DESCRIPTOR), equalTo(UpgradeStatus.NEEDS_MAPPINGS_UPDATE)); } + /** + * Check that the manager will try to upgrade indices where the version in the metadata is null or absent. + */ + public void testManagerProcessesIndicesWithNullVersionMetadata() { + SystemIndices systemIndices = new SystemIndices(org.elasticsearch.common.collect.Map.of("MyIndex", FEATURE)); + SystemIndexManager manager = new SystemIndexManager(systemIndices, client); + + final ClusterState.Builder clusterStateBuilder = createClusterState(Strings.toString(getMappings(null))); + markShardsAvailable(clusterStateBuilder); + + assertThat(manager.getUpgradeStatus(clusterStateBuilder.build(), DESCRIPTOR), equalTo(UpgradeStatus.NEEDS_MAPPINGS_UPDATE)); + } + /** * Check that the manager submits the expected request for an index whose mappings are out-of-date. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index 3b5b37cf980aa..447078ab22de3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -19,6 +19,7 @@ public final class WatcherIndexTemplateRegistryField { // version 11: watch history indices are hidden // version 12: templates changed to composable templates // version 13: add `allow_auto_create` setting + // version 14: remove watches and triggered watches, these are now system indices // Note: if you change this, also inform the kibana team around the watcher-ui public static final int INDEX_TEMPLATE_VERSION = 13; public static final int INDEX_TEMPLATE_VERSION_10 = 10; @@ -32,17 +33,11 @@ public final class WatcherIndexTemplateRegistryField { public static final String HISTORY_TEMPLATE_NAME_NO_ILM_10 = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION_10; public static final String HISTORY_TEMPLATE_NAME_NO_ILM_11 = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION_11; public static final String HISTORY_TEMPLATE_NAME_NO_ILM_12 = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION_12; - public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; - public static final String TRIGGERED_TEMPLATE_NAME_11 = ".triggered_watches-11"; - public static final String TRIGGERED_TEMPLATE_NAME_12 = ".triggered_watches-12"; - public static final String WATCHES_TEMPLATE_NAME = ".watches"; - public static final String WATCHES_TEMPLATE_NAME_11 = ".watches-11"; - public static final String WATCHES_TEMPLATE_NAME_12 = ".watches-12"; public static final String[] TEMPLATE_NAMES = new String[] { - HISTORY_TEMPLATE_NAME, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME + HISTORY_TEMPLATE_NAME }; public static final String[] TEMPLATE_NAMES_NO_ILM = new String[] { - HISTORY_TEMPLATE_NAME_NO_ILM, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME + HISTORY_TEMPLATE_NAME_NO_ILM }; private WatcherIndexTemplateRegistryField() {} diff --git a/x-pack/plugin/core/src/main/resources/triggered-watches.json b/x-pack/plugin/core/src/main/resources/triggered-watches.json deleted file mode 100644 index e11f7d9853186..0000000000000 --- a/x-pack/plugin/core/src/main/resources/triggered-watches.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "index_patterns": [ ".triggered_watches*" ], - "priority": 2147483647, - "template": { - "settings": { - "index.number_of_shards": 1, - "index.auto_expand_replicas": "0-1", - "index.refresh_interval": "-1", - "index.format": 6, - "index.priority": 900 - }, - "mappings": { - "dynamic": "strict", - "properties": { - "trigger_event": { - "type": "object", - "dynamic": true, - "enabled": false, - "properties": { - "schedule": { - "type": "object", - "dynamic": true, - "properties": { - "triggered_time": { - "type": "date" - }, - "scheduled_time": { - "type": "date" - } - } - } - } - }, - "state": { - "type": "keyword" - } - } - } - }, - "allow_auto_create": true, - "_meta": { - "description": "index template for triggered watches indices", - "managed": true - }, - "version": ${xpack.watcher.template.version} -} diff --git a/x-pack/plugin/core/src/main/resources/watches.json b/x-pack/plugin/core/src/main/resources/watches.json deleted file mode 100644 index 4c1108c9f38ad..0000000000000 --- a/x-pack/plugin/core/src/main/resources/watches.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "index_patterns": [ ".watches*" ], - "priority": 2147483647, - "template": { - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0, - "index.auto_expand_replicas": "0-1", - "index.format": 6, - "index.priority": 800 - }, - "mappings": { - "dynamic": "strict", - "properties": { - "status": { - "type": "object", - "enabled": false, - "dynamic": true - }, - "trigger": { - "type": "object", - "enabled": false, - "dynamic": true - }, - "input": { - "type": "object", - "enabled": false, - "dynamic": true - }, - "condition": { - "type": "object", - "enabled": false, - "dynamic": true - }, - "throttle_period": { - "type": "keyword", - "index": false, - "doc_values": false - }, - "throttle_period_in_millis": { - "type": "long", - "index": false, - "doc_values": false - }, - "transform": { - "type": "object", - "enabled": false, - "dynamic": true - }, - "actions": { - "type": "object", - "enabled": false, - "dynamic": true - }, - "metadata": { - "type": "object", - "dynamic": true - } - } - } - }, - "allow_auto_create": true, - "_meta": { - "description": "index template for watches indices", - "managed": true - }, - "version": ${xpack.watcher.template.version} -} diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index 02dce10a1dd69..5c9d9c199afab 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -89,8 +89,6 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME; -import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME; -import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -490,12 +488,6 @@ private void ensureWatcherTemplatesAdded() throws Exception { GetComposableIndexTemplateAction.Response response = client().execute(GetComposableIndexTemplateAction.INSTANCE, new GetComposableIndexTemplateAction.Request(HISTORY_TEMPLATE_NAME)).get(); assertThat("[" + HISTORY_TEMPLATE_NAME + "] is missing", response.indexTemplates().size(), equalTo(1)); - response = client().execute(GetComposableIndexTemplateAction.INSTANCE, - new GetComposableIndexTemplateAction.Request(TRIGGERED_TEMPLATE_NAME)).get(); - assertThat("[" + TRIGGERED_TEMPLATE_NAME + "] is missing", response.indexTemplates().size(), equalTo(1)); - response = client().execute(GetComposableIndexTemplateAction.INSTANCE, - new GetComposableIndexTemplateAction.Request(WATCHES_TEMPLATE_NAME)).get(); - assertThat("[" + WATCHES_TEMPLATE_NAME + "] is missing", response.indexTemplates().size(), equalTo(1)); }); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index f3c3627fa8d76..64d0b85c7ad34 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -17,18 +18,16 @@ import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.client.Client; import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.inject.util.Providers; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.logging.LoggerMessageFormat; -import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -39,6 +38,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; @@ -147,8 +147,8 @@ import org.elasticsearch.xpack.watcher.rest.action.RestDeleteWatchAction; import org.elasticsearch.xpack.watcher.rest.action.RestExecuteWatchAction; import org.elasticsearch.xpack.watcher.rest.action.RestGetWatchAction; -import org.elasticsearch.xpack.watcher.rest.action.RestQueryWatchesAction; import org.elasticsearch.xpack.watcher.rest.action.RestPutWatchAction; +import org.elasticsearch.xpack.watcher.rest.action.RestQueryWatchesAction; import org.elasticsearch.xpack.watcher.rest.action.RestWatchServiceAction; import org.elasticsearch.xpack.watcher.rest.action.RestWatcherStatsAction; import org.elasticsearch.xpack.watcher.support.WatcherIndexTemplateRegistry; @@ -158,13 +158,13 @@ import org.elasticsearch.xpack.watcher.transform.script.WatcherTransformScript; import org.elasticsearch.xpack.watcher.transform.search.SearchTransform; import org.elasticsearch.xpack.watcher.transform.search.SearchTransformFactory; -import org.elasticsearch.xpack.watcher.transport.actions.TransportQueryWatchesAction; import org.elasticsearch.xpack.watcher.transport.actions.TransportAckWatchAction; import org.elasticsearch.xpack.watcher.transport.actions.TransportActivateWatchAction; import org.elasticsearch.xpack.watcher.transport.actions.TransportDeleteWatchAction; import org.elasticsearch.xpack.watcher.transport.actions.TransportExecuteWatchAction; import org.elasticsearch.xpack.watcher.transport.actions.TransportGetWatchAction; import org.elasticsearch.xpack.watcher.transport.actions.TransportPutWatchAction; +import org.elasticsearch.xpack.watcher.transport.actions.TransportQueryWatchesAction; import org.elasticsearch.xpack.watcher.transport.actions.TransportWatcherServiceAction; import org.elasticsearch.xpack.watcher.transport.actions.TransportWatcherStatsAction; import org.elasticsearch.xpack.watcher.trigger.TriggerEngine; @@ -185,8 +185,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.time.Clock; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -205,6 +203,8 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.common.settings.Setting.Property.NodeScope; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.xpack.core.ClientHelper.WATCHER_ORIGIN; public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin, ReloadablePlugin { @@ -247,10 +247,6 @@ public Watcher(final Settings settings) { this.settings = settings; this.transportClient = XPackPlugin.transportClientMode(settings); this.enabled = XPackSettings.WATCHER_ENABLED.get(settings); - - if (enabled && transportClient == false) { - validAutoCreateIndex(settings, logger); - } } // overridable by tests @@ -611,66 +607,6 @@ public void onIndexModule(IndexModule module) { module.addIndexOperationListener(listener); } - static void validAutoCreateIndex(Settings settings, Logger logger) { - String value = settings.get("action.auto_create_index"); - if (value == null) { - return; - } - - String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too" + - " restrictive. disable [action.auto_create_index] or set it to " + - "[{},{},{}*]", (Object) value, Watch.INDEX, TriggeredWatchStoreField.INDEX_NAME, HistoryStoreField.INDEX_PREFIX); - if (Booleans.isFalse(value)) { - throw new IllegalArgumentException(errorMessage); - } - - if (Booleans.isTrue(value)) { - return; - } - - String[] matches = Strings.commaDelimitedListToStringArray(value); - List indices = new ArrayList<>(); - indices.add(".watches"); - indices.add(".triggered_watches"); - ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); - indices.add(HistoryStoreField.getHistoryIndexNameForTime(now, null)); - indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusDays(1), null)); - indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(1), null)); - indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(2), null)); - indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(3), null)); - indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(4), null)); - indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(5), null)); - indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(6), null)); - for (String index : indices) { - boolean matched = false; - for (String match : matches) { - char c = match.charAt(0); - if (c == '-') { - if (Regex.simpleMatch(match.substring(1), index)) { - throw new IllegalArgumentException(errorMessage); - } - } else if (c == '+') { - if (Regex.simpleMatch(match.substring(1), index)) { - matched = true; - break; - } - } else { - if (Regex.simpleMatch(match, index)) { - matched = true; - break; - } - } - } - if (matched == false) { - throw new IllegalArgumentException(errorMessage); - } - } - logger.warn("the [action.auto_create_index] setting is configured to be restrictive [{}]. " + - " for the next 6 months daily history indices are allowed to be created, but please make sure" + - " that any future history indices after 6 months with the pattern " + - "[.watcher-history-yyyy.MM.dd] are allowed to be created", value); - } - // These are all old templates from pre 6.0 era, that need to be deleted @Override public UnaryOperator> getIndexTemplateMetadataUpgrader() { @@ -718,10 +654,26 @@ public void reload(Settings settings) { @Override public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.unmodifiableList(Arrays.asList( - new SystemIndexDescriptor(Watch.INDEX, "Contains Watch definitions"), - new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, "Used to track current and queued Watch execution") - )); + return Arrays.asList( + SystemIndexDescriptor.builder() + .setIndexPattern(Watch.INDEX + "*") + .setPrimaryIndex(Watch.INDEX) + .setDescription("Contains Watch definitions") + .setMappings(getWatchesIndexMappings()) + .setSettings(getWatchesIndexSettings()) + .setVersionMetaKey("version") + .setOrigin(WATCHER_ORIGIN) + .build(), + SystemIndexDescriptor.builder() + .setIndexPattern(TriggeredWatchStoreField.INDEX_NAME + "*") + .setPrimaryIndex(TriggeredWatchStoreField.INDEX_NAME) + .setDescription("Used to track current and queued Watch execution") + .setMappings(getTriggeredWatchesIndexMappings()) + .setSettings(getTriggeredWatchesIndexSettings()) + .setVersionMetaKey("version") + .setOrigin(WATCHER_ORIGIN) + .build() + ); } @Override @@ -733,4 +685,166 @@ public String getFeatureName() { public String getFeatureDescription() { return "Manages Watch definitions and state"; } + + private Settings getWatchesIndexSettings() { + return Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.auto_expand_replicas", "0-1") + .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), 6) + .put(IndexMetadata.SETTING_PRIORITY, 800) + .build(); + } + + private XContentBuilder getWatchesIndexMappings() { + try { + final XContentBuilder builder = jsonBuilder(); + + builder.startObject(); + { + builder.startObject(SINGLE_MAPPING_NAME); + builder.field("dynamic", "strict"); + { + builder.startObject("_meta"); + builder.field("version", Version.CURRENT); + builder.endObject(); + } + { + builder.startObject("properties"); + { + builder.startObject("status"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.field("dynamic", true); + builder.endObject(); + + builder.startObject("trigger"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.field("dynamic", true); + builder.endObject(); + + builder.startObject("input"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.field("dynamic", true); + builder.endObject(); + + builder.startObject("condition"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.field("dynamic", true); + builder.endObject(); + + builder.startObject("throttle_period"); + builder.field("type", "keyword"); + builder.field("index", false); + builder.field("doc_values", false); + builder.endObject(); + + builder.startObject("throttle_period_in_millis"); + builder.field("type", "long"); + builder.field("index", false); + builder.field("doc_values", false); + builder.endObject(); + + builder.startObject("transform"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.field("dynamic", true); + builder.endObject(); + + builder.startObject("actions"); + builder.field("type", "object"); + builder.field("enabled", false); + builder.field("dynamic", true); + builder.endObject(); + + builder.startObject("metadata"); + builder.field("type", "object"); + builder.field("dynamic", true); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + + builder.endObject(); + return builder; + } catch (IOException e) { + throw new UncheckedIOException("Failed to build " + Watch.INDEX + " index mappings", e); + } + } + + private Settings getTriggeredWatchesIndexSettings() { + return Settings.builder() + .put("index.number_of_shards", 1) + .put("index.auto_expand_replicas", "0-1") + .put("index.refresh_interval", "-1") + .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), 6) + .put(IndexMetadata.SETTING_PRIORITY, 900) + .build(); + } + + private XContentBuilder getTriggeredWatchesIndexMappings() { + try { + final XContentBuilder builder = jsonBuilder(); + + builder.startObject(); + { + builder.startObject(SINGLE_MAPPING_NAME); + builder.field("dynamic", "strict"); + { + builder.startObject("_meta"); + builder.field("version", Version.CURRENT); + builder.endObject(); + } + { + builder.startObject("properties"); + { + builder.startObject("trigger_event"); + { + builder.field("type", "object"); + builder.field("dynamic", true); + builder.field("enabled", false); + builder.startObject("properties"); + { + builder.startObject("schedule"); + { + builder.field("type", "object"); + builder.field("dynamic", true); + builder.startObject("properties"); + { + builder.startObject("triggered_time"); + builder.field("type", "date"); + builder.endObject(); + + builder.startObject("scheduled_time"); + builder.field("type", "date"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + + builder.startObject("state"); + builder.field("type", "keyword"); + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + + builder.endObject(); + return builder; + } catch (IOException e) { + throw new UncheckedIOException("Failed to build " + TriggeredWatchStoreField.INDEX_NAME + " index mappings", e); + } + } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java index 287ebabe0f75c..e0469159afff7 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField; import org.elasticsearch.xpack.watcher.Watcher; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -28,16 +27,6 @@ public class WatcherIndexTemplateRegistry extends IndexTemplateRegistry { public static final String WATCHER_TEMPLATE_VERSION_VARIABLE = "xpack.watcher.template.version"; - public static final IndexTemplateConfig TEMPLATE_CONFIG_TRIGGERED_WATCHES = new IndexTemplateConfig( - WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, - "/triggered-watches.json", - WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION, - WATCHER_TEMPLATE_VERSION_VARIABLE); - public static final IndexTemplateConfig TEMPLATE_CONFIG_TRIGGERED_WATCHES_11 = new IndexTemplateConfig( - WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME_11, - "/triggered-watches-11.json", - 11, - WATCHER_TEMPLATE_VERSION_VARIABLE); public static final IndexTemplateConfig TEMPLATE_CONFIG_WATCH_HISTORY = new IndexTemplateConfig( WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, "/watch-history.json", @@ -68,16 +57,6 @@ public class WatcherIndexTemplateRegistry extends IndexTemplateRegistry { "/watch-history-no-ilm-11.json", 11, WATCHER_TEMPLATE_VERSION_VARIABLE); - public static final IndexTemplateConfig TEMPLATE_CONFIG_WATCHES = new IndexTemplateConfig( - WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, - "/watches.json", - WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION, - WATCHER_TEMPLATE_VERSION_VARIABLE); - public static final IndexTemplateConfig TEMPLATE_CONFIG_WATCHES_11 = new IndexTemplateConfig( - WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME_11, - "/watches-11.json", - 11, - WATCHER_TEMPLATE_VERSION_VARIABLE); public static final LifecyclePolicyConfig POLICY_WATCH_HISTORY = new LifecyclePolicyConfig("watch-history-ilm-policy", "/watch-history-ilm-policy.json"); @@ -95,26 +74,20 @@ protected List getLegacyTemplateConfigs() { if (clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_7_9_0)) { return Collections.emptyList(); } else if (clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) { - return Arrays.asList( - ilmManagementEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY_11 : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM_11, - TEMPLATE_CONFIG_TRIGGERED_WATCHES_11, - TEMPLATE_CONFIG_WATCHES_11 + return Collections.singletonList( + ilmManagementEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY_11 : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM_11 ); } else { - return Arrays.asList( - ilmManagementEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY_10 : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM_10, - TEMPLATE_CONFIG_TRIGGERED_WATCHES_11, - TEMPLATE_CONFIG_WATCHES_11 + return Collections.singletonList( + ilmManagementEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY_10 : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM_10 ); } } @Override protected List getComposableTemplateConfigs() { - return Arrays.asList( - ilmManagementEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM, - TEMPLATE_CONFIG_TRIGGERED_WATCHES, - TEMPLATE_CONFIG_WATCHES + return Collections.singletonList( + ilmManagementEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM ); } @@ -135,22 +108,18 @@ protected String getOrigin() { } public static boolean validate(ClusterState state) { - if(state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_9_0)){ + if (state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_9_0)){ return (state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) || - state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) && - state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) && - state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME); - } else if (state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) { + state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)); + } + + if (state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) { return (state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_11) || - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_11)) && - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME_11) && - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME_11); - } else { - return (state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10) || - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_10)) && - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME_11) && - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME_11); + state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_11)); } + + return (state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10) || + state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_10)); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java index a64bd7be53005..0e2122a0a3463 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java @@ -48,8 +48,6 @@ import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME; -import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME; -import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; @@ -95,8 +93,6 @@ public void testNoRestartWithoutAllocationIdsConfigured() { ClusterState clusterState = ClusterState.builder(new ClusterName("my-cluster")) .metadata(Metadata.builder() .put(IndexTemplateMetadata.builder(HISTORY_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(TRIGGERED_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(WATCHES_TEMPLATE_NAME).patterns(randomIndexPatterns())) .build()) .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1"))) .routingTable(RoutingTable.builder().add(watchRoutingTable).build()) @@ -127,8 +123,6 @@ public void testShutdown() { .routingTable(RoutingTable.builder().add(watchRoutingTable).build()) .metadata(Metadata.builder() .put(IndexTemplateMetadata.builder(HISTORY_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(TRIGGERED_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(WATCHES_TEMPLATE_NAME).patterns(randomIndexPatterns())) .build()) .build(); @@ -153,9 +147,7 @@ public void testManualStartStop() { .numberOfShards(1).numberOfReplicas(0); Metadata.Builder metadataBuilder = Metadata.builder() .put(indexMetadataBuilder) - .put(IndexTemplateMetadata.builder(HISTORY_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(TRIGGERED_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(WATCHES_TEMPLATE_NAME).patterns(randomIndexPatterns())); + .put(IndexTemplateMetadata.builder(HISTORY_TEMPLATE_NAME).patterns(randomIndexPatterns())); if (randomBoolean()) { metadataBuilder.putCustom(WatcherMetadata.TYPE, new WatcherMetadata(false)); } @@ -428,17 +420,6 @@ public void testWatcherServiceDoesNotStartIfIndexTemplatesAreMissing() throws Ex if (isHistoryTemplateAdded) { metadataBuilder.put(IndexTemplateMetadata.builder(HISTORY_TEMPLATE_NAME).patterns(randomIndexPatterns())); } - boolean isTriggeredTemplateAdded = randomBoolean(); - if (isTriggeredTemplateAdded) { - metadataBuilder.put(IndexTemplateMetadata.builder(TRIGGERED_TEMPLATE_NAME).patterns(randomIndexPatterns())); - } - boolean isWatchesTemplateAdded = randomBoolean(); - if (isWatchesTemplateAdded) { - // ensure not all templates are added, otherwise life cycle service would start - if ((isHistoryTemplateAdded || isTriggeredTemplateAdded) == false) { - metadataBuilder.put(IndexTemplateMetadata.builder(WATCHES_TEMPLATE_NAME).patterns(randomIndexPatterns())); - } - } ClusterState state = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).metadata(metadataBuilder).build(); when(watcherService.validate(eq(state))).thenReturn(true); @@ -486,8 +467,6 @@ public void testMasterOnlyNodeCanStart() { public void testDataNodeWithoutDataCanStart() { Metadata metadata = Metadata.builder().put(IndexTemplateMetadata.builder(HISTORY_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(TRIGGERED_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(WATCHES_TEMPLATE_NAME).patterns(randomIndexPatterns())) .build(); ClusterState state = ClusterState.builder(new ClusterName("my-cluster")) .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1"))) @@ -567,8 +546,7 @@ private void startWatcher() { .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), 6)) // the internal index format, required .numberOfShards(1).numberOfReplicas(0); Metadata metadata = Metadata.builder().put(IndexTemplateMetadata.builder(HISTORY_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(TRIGGERED_TEMPLATE_NAME).patterns(randomIndexPatterns())) - .put(IndexTemplateMetadata.builder(WATCHES_TEMPLATE_NAME).patterns(randomIndexPatterns())).put(indexMetadataBuilder) + .put(indexMetadataBuilder) .build(); ClusterState state = ClusterState.builder(new ClusterName("my-cluster")) .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1") diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java index 3c513963b77d6..6b8cde1020b25 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java @@ -26,7 +26,6 @@ import java.util.List; import static java.util.Collections.emptyMap; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.mock; @@ -36,33 +35,6 @@ public class WatcherPluginTests extends ESTestCase { - public void testValidAutoCreateIndex() { - Watcher.validAutoCreateIndex(Settings.EMPTY, logger); - Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", true).build(), logger); - - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", false).build(), logger)); - assertThat(exception.getMessage(), containsString("[.watches,.triggered_watches,.watcher-history-*]")); - - Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", - ".watches,.triggered_watches,.watcher-history*").build(), logger); - Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", "*w*").build(), logger); - Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".w*,.t*").build(), logger); - - exception = expectThrows(IllegalArgumentException.class, - () -> Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watches").build(), logger)); - assertThat(exception.getMessage(), containsString("[.watches,.triggered_watches,.watcher-history-*]")); - - exception = expectThrows(IllegalArgumentException.class, - () -> Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".triggered_watch").build(), logger)); - assertThat(exception.getMessage(), containsString("[.watches,.triggered_watches,.watcher-history-*]")); - - exception = expectThrows(IllegalArgumentException.class, - () -> Watcher.validAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".watcher-history-*").build(), - logger)); - assertThat(exception.getMessage(), containsString("[.watches,.triggered_watches,.watcher-history-*]")); - } - public void testWatcherDisabledTests() throws Exception { Settings settings = Settings.builder() .put("xpack.watcher.enabled", false) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java index 80a993b977127..38303384aa70c 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -122,16 +122,15 @@ public void testThatNonExistingTemplatesAreAddedImmediately() { registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class); - verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); + verify(client, times(1)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); // now delete one template from the cluster state and lets retry Map existingTemplates = new HashMap<>(); existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); - existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(newEvent); argumentCaptor = ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class); - verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); + verify(client, times(1)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); PutComposableIndexTemplateAction.Request req = argumentCaptor.getAllValues().stream() .filter(r -> r.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME)) .findFirst() @@ -150,16 +149,15 @@ public void testThatNonExistingTemplatesAreAddedEvenWithILMUsageDisabled() { registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class); - verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); + verify(client, times(1)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); // now delete one template from the cluster state and lets retry Map existingTemplates = new HashMap<>(); existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); - existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(newEvent); ArgumentCaptor captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); - verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); + verify(client, times(1)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); captor.getAllValues().forEach(req -> assertNull(req.settings().get("index.lifecycle.name"))); verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } @@ -264,15 +262,13 @@ public void testThatTemplatesAreAppliedOnNewerNodes() { DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build(); Map existingTemplates = new HashMap<>(); - existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); - existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); existingTemplates.put(".watch-history-6", 6); ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class); - verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); + verify(client, times(1)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); assertTrue(argumentCaptor.getAllValues().stream() .anyMatch(r -> r.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME))); } @@ -283,8 +279,6 @@ public void testThatTemplatesWithHiddenAreAppliedOnNewerNodes() { DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("master").masterNodeId("master").add(node).add(masterNode).build(); Map existingTemplates = new HashMap<>(); - existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); - existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); existingTemplates.put(".watch-history-6", 6); ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); @@ -320,8 +314,6 @@ public void testThatTemplatesAreNotAppliedOnSameVersionNodes() { DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build(); Map existingTemplates = new HashMap<>(); - existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, null); - existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, null); existingTemplates.put(".watch-history-6", null); ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); @@ -334,8 +326,6 @@ public void testThatMissingMasterNodeDoesNothing() { DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); Map existingTemplates = new HashMap<>(); - existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, null); - existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, null); existingTemplates.put(".watch-history-6", null); ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java index 24ec1d6e5f34e..f96c118486870 100644 --- a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java @@ -15,11 +15,9 @@ public final class XPackRestTestConstants { // Watcher constants: public static final String INDEX_TEMPLATE_VERSION = "10"; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; - public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; - public static final String WATCHES_TEMPLATE_NAME = ".watches"; public static final String[] TEMPLATE_NAMES_NO_ILM = new String[] { - HISTORY_TEMPLATE_NAME_NO_ILM, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME + HISTORY_TEMPLATE_NAME_NO_ILM }; // ML constants: