From ab9dfdb4afb2ae15c7b7b0cef3dbcc252b6cc823 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 27 Jan 2021 12:20:38 +0000 Subject: [PATCH 1/8] [ML] Automatic management for ML system indices The ML system indices now use the special functionality for applying the correct mappings on first use. This replaces the index templates that used to do this job, but were vulnerable to tampering. --- .../indices/SystemIndexDescriptor.java | 10 +- .../xpack/core/ml/MlConfigIndex.java | 13 ++ .../xpack/core/ml/MlMetaIndex.java | 20 +++ .../persistence/InferenceIndexConstants.java | 20 +++ .../persistence/AnomalyDetectorsIndex.java | 2 - .../xpack/core/ml/config_index_template.json | 15 -- .../core/ml/inference_index_mappings.json | 142 ++++++++++++++++ .../core/ml/inference_index_template.json | 156 ------------------ .../xpack/core/ml/meta_index_mappings.json | 34 ++++ .../xpack/core/ml/meta_index_template.json | 47 ------ .../core/ml/utils/MlIndexAndAliasTests.java | 6 +- .../xpack/ml/MachineLearning.java | 41 +++-- .../xpack/ml/MlConfigMigrator.java | 2 +- .../xpack/ml/MlIndexTemplateRegistry.java | 29 ---- ...ransportStartDataFrameAnalyticsAction.java | 25 +-- .../persistence/DatafeedConfigProvider.java | 7 +- .../ml/job/persistence/JobConfigProvider.java | 11 +- .../ml/MlIndexTemplateRegistryTests.java | 4 +- ...ortStartDataFrameAnalyticsActionTests.java | 4 +- .../test/rest/AbstractXPackRestTest.java | 2 - ...nfigIndexMappingsFullClusterRestartIT.java | 4 +- .../MlMigrationFullClusterRestartIT.java | 5 +- .../upgrades/MlJobSnapshotUpgradeIT.java | 5 +- .../upgrades/MlMappingsUpgradeIT.java | 6 +- .../UpgradeClusterClientYamlTestSuiteIT.java | 5 +- .../rest/IndexMappingTemplateAsserter.java | 48 +++--- .../test/rest/XPackRestTestConstants.java | 5 + 27 files changed, 336 insertions(+), 332 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_mappings.json delete mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json delete mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 7c9922d024afa..125bbbae330cc 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -242,7 +242,7 @@ public static class Builder { private String indexPattern; private String primaryIndex; private String description; - private XContentBuilder mappingsBuilder = null; + private String mappings = null; private Settings settings = null; private String aliasName = null; private int indexFormat = 0; @@ -267,7 +267,12 @@ public Builder setDescription(String description) { } public Builder setMappings(XContentBuilder mappingsBuilder) { - this.mappingsBuilder = mappingsBuilder; + mappings = mappingsBuilder == null ? null : Strings.toString(mappingsBuilder); + return this; + } + + public Builder setMappings(String mappings) { + this.mappings = mappings; return this; } @@ -297,7 +302,6 @@ public Builder setOrigin(String origin) { } public SystemIndexDescriptor build() { - String mappings = mappingsBuilder == null ? null : Strings.toString(mappingsBuilder); return new SystemIndexDescriptor( indexPattern, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java index 085a75840a665..c0e7803e83661 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java @@ -6,6 +6,9 @@ package org.elasticsearch.xpack.core.ml; import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xpack.core.template.TemplateUtils; public final class MlConfigIndex { @@ -13,6 +16,8 @@ public final class MlConfigIndex { private static final String INDEX_NAME = ".ml-config"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000; + /** * The name of the index where job, datafeed and analytics configuration is stored * @@ -29,5 +34,13 @@ public static String mapping() { MAPPINGS_VERSION_VARIABLE); } + public static Settings settings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), CONFIG_INDEX_MAX_RESULTS_WINDOW) + .build(); + } + private MlConfigIndex() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index 9c969aea80bbd..9157f14b85bd6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -5,9 +5,15 @@ */ package org.elasticsearch.xpack.core.ml; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.core.template.TemplateUtils; + public final class MlMetaIndex { private static final String INDEX_NAME = ".ml-meta"; + private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; /** * Where to store the ml info in Elasticsearch - must match what's @@ -19,5 +25,19 @@ public static String indexName() { return INDEX_NAME; } + public static String mapping() { + return TemplateUtils.loadTemplate( + "/org/elasticsearch/xpack/core/ml/meta_index_mappings.json", + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE); + } + + public static Settings settings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .build(); + } + private MlMetaIndex() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java index 9d1f2ff276a85..3221f6a683a3c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java @@ -5,7 +5,11 @@ */ package org.elasticsearch.xpack.core.ml.inference.persistence; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.core.template.TemplateUtils; /** * Class containing the index constants so that the index version, name, and prefix are available to a wider audience. @@ -28,5 +32,21 @@ public final class InferenceIndexConstants { public static final String LATEST_INDEX_NAME = INDEX_NAME_PREFIX + INDEX_VERSION; public static final ParseField DOC_TYPE = new ParseField("doc_type"); + private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + + public static String mapping() { + return TemplateUtils.loadTemplate( + "/org/elasticsearch/xpack/core/ml/inference_index_mappings.json", + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE); + } + + public static Settings settings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .build(); + } + private InferenceIndexConstants() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java index fc8e31bc4649f..a0e886f5b1b8b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java @@ -18,8 +18,6 @@ */ public final class AnomalyDetectorsIndex { - public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000; - private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/"; diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json deleted file mode 100644 index 8c6c352ab245a..0000000000000 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "order" : 0, - "version" : ${xpack.ml.version.id}, - "index_patterns" : [ - ".ml-config" - ], - "settings" : { - "index" : { - "max_result_window" : "${xpack.ml.config.max_result_window}", - "number_of_shards" : "1", - "auto_expand_replicas" : "0-1" - } - }, - "mappings": ${xpack.ml.config.mappings} -} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_mappings.json new file mode 100644 index 0000000000000..171cbabc52c30 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_mappings.json @@ -0,0 +1,142 @@ +{ + "_doc": { + "_meta": { + "version": "${xpack.ml.version}" + }, + "dynamic": false, + "properties": { + "doc_type": { + "type": "keyword" + }, + "model_id": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + }, + "input": { + "enabled": false + }, + "version": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "create_time": { + "type": "date" + }, + "tags": { + "type": "keyword" + }, + "metadata": { + "enabled": false + }, + "estimated_operations": { + "type": "long" + }, + "estimated_heap_memory_usage_bytes": { + "type": "long" + }, + "doc_num": { + "type": "long" + }, + "definition": { + "enabled": false + }, + "compression_version": { + "type": "long" + }, + "definition_length": { + "type": "long" + }, + "total_definition_length": { + "type": "long" + }, + "default_field_map": { + "enabled": false + }, + "inference_config": { + "enabled": false + }, + "feature_importance_baseline": { + "properties": { + "baseline": { + "type": "double" + }, + "classes": { + "properties": { + "class_name": { "type": "keyword"}, + "baseline": {"type" : "double"} + } + } + } + }, + "total_feature_importance": { + "type": "nested", + "dynamic": false, + "properties": { + "importance": { + "properties": { + "min": { + "type": "double" + }, + "max": { + "type": "double" + }, + "mean_magnitude": { + "type": "double" + } + } + }, + "feature_name": { + "type": "keyword" + }, + "classes": { + "type": "nested", + "dynamic": false, + "properties": { + "importance": { + "properties": { + "min": { + "type": "double" + }, + "max": { + "type": "double" + }, + "mean_magnitude": { + "type": "double" + } + } + }, + "class_name": { + "type": "keyword" + } + } + } + } + }, + "hyperparameters": { + "type": "nested", + "dynamic": false, + "properties": { + "name": { + "type": "keyword" + }, + "value": { + "type": "double" + }, + "absolute_importance": { + "type": "double" + }, + "relative_importance": { + "type": "double" + }, + "supplied": { + "type": "boolean" + } + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json deleted file mode 100644 index a1876b5e75bb2..0000000000000 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "order" : 0, - "version" : ${xpack.ml.version.id}, - "index_patterns" : [ - ".ml-inference-000003" - ], - "settings" : { - "index" : { - "number_of_shards" : "1", - "auto_expand_replicas" : "0-1" - } - }, - "mappings" : { - "_doc": { - "_meta": { - "version": "${xpack.ml.version}" - }, - "dynamic": "false", - "properties": { - "doc_type": { - "type": "keyword" - }, - "model_id": { - "type": "keyword" - }, - "created_by": { - "type": "keyword" - }, - "input": { - "enabled": false - }, - "version": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "create_time": { - "type": "date" - }, - "tags": { - "type": "keyword" - }, - "metadata": { - "enabled": false - }, - "estimated_operations": { - "type": "long" - }, - "estimated_heap_memory_usage_bytes": { - "type": "long" - }, - "doc_num": { - "type": "long" - }, - "definition": { - "enabled": false - }, - "compression_version": { - "type": "long" - }, - "definition_length": { - "type": "long" - }, - "total_definition_length": { - "type": "long" - }, - "default_field_map": { - "enabled": false - }, - "inference_config": { - "enabled": false - }, - "feature_importance_baseline": { - "properties": { - "baseline": { - "type": "double" - }, - "classes": { - "properties": { - "class_name": { "type": "keyword"}, - "baseline": {"type" : "double"} - } - } - } - }, - "total_feature_importance": { - "type": "nested", - "dynamic": "false", - "properties": { - "importance": { - "properties": { - "min": { - "type": "double" - }, - "max": { - "type": "double" - }, - "mean_magnitude": { - "type": "double" - } - } - }, - "feature_name": { - "type": "keyword" - }, - "classes": { - "type": "nested", - "dynamic": "false", - "properties": { - "importance": { - "properties": { - "min": { - "type": "double" - }, - "max": { - "type": "double" - }, - "mean_magnitude": { - "type": "double" - } - } - }, - "class_name": { - "type": "keyword" - } - } - } - } - }, - "hyperparameters": { - "type": "nested", - "dynamic": "false", - "properties": { - "name": { - "type": "keyword" - }, - "value": { - "type": "double" - }, - "absolute_importance": { - "type": "double" - }, - "relative_importance": { - "type": "double" - }, - "supplied": { - "type": "boolean" - } - } - } - } - } - }, - "aliases" : { } -} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json new file mode 100644 index 0000000000000..4f038650f9ad7 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json @@ -0,0 +1,34 @@ +{ + "_doc": { + "_meta": { + "version": "${xpack.ml.version}" + }, + "dynamic_templates": [ + { + "strings_as_keywords": { + "match": "*", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "calendar_id": { + "type": "keyword" + }, + "job_ids": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "start_time": { + "type": "date" + }, + "end_time": { + "type": "date" + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json deleted file mode 100644 index 19df45c52509c..0000000000000 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "order" : 0, - "version" : ${xpack.ml.version.id}, - "index_patterns" : [ - ".ml-meta" - ], - "settings" : { - "index" : { - "number_of_shards" : "1", - "auto_expand_replicas" : "0-1" - } - }, - "mappings" : { - "_doc": { - "_meta": { - "version": "${xpack.ml.version}" - }, - "dynamic_templates": [ - { - "strings_as_keywords": { - "match": "*", - "mapping": { - "type": "keyword" - } - } - } - ], - "properties": { - "calendar_id": { - "type": "keyword" - }, - "job_ids": { - "type": "keyword" - }, - "description": { - "type": "keyword" - }, - "start_time": { - "type": "date" - }, - "end_time": { - "type": "date" - } - } - } - } -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index 9cb6b708827b3..a99427daa947e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -138,11 +138,11 @@ public void testInstallIndexTemplateIfRequired_GivenTemplateExists() { public void testInstallIndexTemplateIfRequired() { ClusterState clusterState = createClusterState(Collections.emptyMap()); - IndexTemplateConfig inferenceTemplate = new IndexTemplateConfig(InferenceIndexConstants.LATEST_INDEX_NAME, - "/org/elasticsearch/xpack/core/ml/inference_index_template.json", Version.CURRENT.id, "xpack.ml.version", + IndexTemplateConfig notificationsTemplate = new IndexTemplateConfig(InferenceIndexConstants.LATEST_INDEX_NAME, + "/org/elasticsearch/xpack/core/ml/notifications_index_template.json", Version.CURRENT.id, "xpack.ml.version", Collections.singletonMap("xpack.ml.version.id", String.valueOf(Version.CURRENT.id))); - MlIndexAndAlias.installIndexTemplateIfRequired(clusterState, client, inferenceTemplate, listener); + MlIndexAndAlias.installIndexTemplateIfRequired(clusterState, client, notificationsTemplate, listener); InOrder inOrder = inOrder(indicesAdminClient, listener); inOrder.verify(indicesAdminClient).putTemplate(any(), any()); inOrder.verify(listener).onResponse(true); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 4c19a1da631d8..c0a17570243a6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -69,7 +69,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService; -import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; @@ -358,6 +357,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; public class MachineLearning extends Plugin implements SystemIndexPlugin, AnalysisPlugin, @@ -636,7 +636,7 @@ public Collection createComponents(Client client, ClusterService cluster DataFrameAnalyticsAuditor dataFrameAnalyticsAuditor = new DataFrameAnalyticsAuditor(client, clusterService); InferenceAuditor inferenceAuditor = new InferenceAuditor(client, clusterService); this.dataFrameAnalyticsAuditor.set(dataFrameAnalyticsAuditor); - OriginSettingClient originSettingClient = new OriginSettingClient(client, ClientHelper.ML_ORIGIN); + OriginSettingClient originSettingClient = new OriginSettingClient(client, ML_ORIGIN); ResultsPersisterService resultsPersisterService = new ResultsPersisterService( threadPool, originSettingClient, @@ -853,8 +853,7 @@ public List> getPersistentTasksExecutor(ClusterServic dataFrameAnalyticsManager.get(), dataFrameAnalyticsAuditor.get(), memoryTracker.get(), - expressionResolver, - MlIndexTemplateRegistry.INFERENCE_TEMPLATE), + expressionResolver), new SnapshotUpgradeTaskExecutor(settings, clusterService, autodetectProcessManager.get(), @@ -1084,10 +1083,8 @@ public static boolean allTemplatesInstalled(ClusterState clusterState) { List templateNames = Arrays.asList( NotificationsIndex.NOTIFICATIONS_INDEX, - MlMetaIndex.indexName(), AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, - AnomalyDetectorsIndex.jobResultsIndexPrefix(), - InferenceIndexConstants.LATEST_INDEX_NAME); + AnomalyDetectorsIndex.jobResultsIndexPrefix()); for (String templateName : templateNames) { allPresent = allPresent && TemplateUtils.checkTemplateExistsAndVersionIsGTECurrentVersion(templateName, clusterState); } @@ -1164,9 +1161,33 @@ public List getNamedWriteables() { @Override public Collection getSystemIndexDescriptors(Settings settings) { return List.of( - new SystemIndexDescriptor(MlMetaIndex.indexName(), "Contains scheduling and anomaly tracking metadata"), - new SystemIndexDescriptor(MlConfigIndex.indexName(), "Contains ML configuration data"), - new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, "Contains ML model configuration and statistics") + SystemIndexDescriptor.builder() + .setIndexPattern(MlMetaIndex.indexName() + "*") + .setPrimaryIndex(MlMetaIndex.indexName()) + .setDescription("Contains scheduling and anomaly tracking metadata") + .setMappings(MlMetaIndex.mapping()) + .setSettings(MlMetaIndex.settings()) + .setVersionMetaKey("version") + .setOrigin(ML_ORIGIN) + .build(), + SystemIndexDescriptor.builder() + .setIndexPattern(MlConfigIndex.indexName() + "*") + .setPrimaryIndex(MlConfigIndex.indexName()) + .setDescription("Contains ML configuration data") + .setMappings(MlConfigIndex.mapping()) + .setSettings(MlConfigIndex.settings()) + .setVersionMetaKey("version") + .setOrigin(ML_ORIGIN) + .build(), + SystemIndexDescriptor.builder() + .setIndexPattern(InferenceIndexConstants.INDEX_PATTERN) + .setPrimaryIndex(InferenceIndexConstants.LATEST_INDEX_NAME) + .setDescription("Contains ML model configuration and statistics") + .setMappings(InferenceIndexConstants.mapping()) + .setSettings(InferenceIndexConstants.settings()) + .setVersionMetaKey("version") + .setOrigin(ML_ORIGIN) + .build() ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index d5cca467d2b30..3e65f51381819 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -495,7 +495,7 @@ private void createConfigIndex(ActionListener listener) { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") - .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) + .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) ); createIndexRequest.mapping(MlConfigIndex.mapping()); } catch (Exception e) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java index 5a2ede6c5fcaf..7e168a82004ba 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java @@ -12,10 +12,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ClientHelper; -import org.elasticsearch.xpack.core.ml.MlConfigIndex; -import org.elasticsearch.xpack.core.ml.MlMetaIndex; import org.elasticsearch.xpack.core.ml.MlStatsIndex; -import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; @@ -42,39 +39,16 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE = stateTemplate(); - private static final IndexTemplateConfig META_TEMPLATE = new IndexTemplateConfig(MlMetaIndex.indexName(), - ROOT_RESOURCE_PATH + "meta_index_template.json", Version.CURRENT.id, VERSION_PATTERN, - Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id))); - public static final IndexTemplateConfig NOTIFICATIONS_TEMPLATE = new IndexTemplateConfig(NotificationsIndex.NOTIFICATIONS_INDEX, ROOT_RESOURCE_PATH + "notifications_index_template.json", Version.CURRENT.id, VERSION_PATTERN, Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id))); - private static final IndexTemplateConfig CONFIG_TEMPLATE = configTemplate(); - - public static final IndexTemplateConfig INFERENCE_TEMPLATE = new IndexTemplateConfig(InferenceIndexConstants.LATEST_INDEX_NAME, - ROOT_RESOURCE_PATH + "inference_index_template.json", Version.CURRENT.id, VERSION_PATTERN, - Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id))); - private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate(); private static final String ML_SIZE_BASED_ILM_POLICY_NAME = "ml-size-based-ilm-policy"; private static final LifecyclePolicyConfig ML_SIZE_BASED_ILM_POLICY = new LifecyclePolicyConfig(ML_SIZE_BASED_ILM_POLICY_NAME, ROOT_RESOURCE_PATH + "size_based_ilm_policy.json"); - private static IndexTemplateConfig configTemplate() { - Map variables = new HashMap<>(); - variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); - variables.put("xpack.ml.config.max_result_window", - String.valueOf(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)); - variables.put("xpack.ml.config.mappings", MlConfigIndex.mapping()); - - return new IndexTemplateConfig(MlConfigIndex.indexName(), - ROOT_RESOURCE_PATH + "config_index_template.json", - Version.CURRENT.id, VERSION_PATTERN, - variables); - } - private static IndexTemplateConfig stateTemplate() { Map variables = new HashMap<>(); variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); @@ -119,9 +93,6 @@ public MlIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterServ templatesToUse = Arrays.asList( ANOMALY_DETECTION_RESULTS_TEMPLATE, ANOMALY_DETECTION_STATE_TEMPLATE, - CONFIG_TEMPLATE, - INFERENCE_TEMPLATE, - META_TEMPLATE, NOTIFICATIONS_TEMPLATE, STATS_TEMPLATE); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index 4db85ff2246d6..955e055bfb2de 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -67,9 +67,7 @@ import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.core.ml.utils.PhaseProgress; -import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager; import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsTask; @@ -592,13 +590,11 @@ public static class TaskExecutor extends AbstractJobPersistentTasksExecutor clusterState = event.state()); } @@ -681,22 +676,8 @@ protected void nodeOperation(AllocatedPersistentTask task, TaskParams params, Pe ); // Get stats to initialize in memory stats tracking - ActionListener templateCheckListener = ActionListener.wrap( - ok -> executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsStatsAction.INSTANCE, - new GetDataFrameAnalyticsStatsAction.Request(params.getId()), statsListener), - error -> { - Throwable cause = ExceptionsHelper.unwrapCause(error); - logger.error( - new ParameterizedMessage( - "[{}] failed to create internal index template [{}]", - params.getId(), - inferenceIndexTemplate.getTemplateName()), - cause); - dfaTask.setFailed(error); - } - ); - - MlIndexAndAlias.installIndexTemplateIfRequired(clusterState, client, inferenceIndexTemplate, templateCheckListener); + executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsStatsAction.INSTANCE, + new GetDataFrameAnalyticsStatsAction.Request(params.getId()), statsListener); } private void searchProgressFromIndex(String jobId, ActionListener listener) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index 70aeda37aea5f..96b49bb7ac9cf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -51,7 +51,6 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.MlStrings; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; @@ -77,7 +76,7 @@ * datafeed configuration document * * The number of datafeeds returned in a search it limited to - * {@link AnomalyDetectorsIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}. + * {@link MlConfigIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}. * In most cases we expect 10s or 100s of datafeeds to be defined and * a search for all datafeeds should return all. */ @@ -368,7 +367,7 @@ public void expandDatafeedIds(String expression, SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) .setSource(sourceBuilder) - .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) + .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch); @@ -420,7 +419,7 @@ public void expandDatafeedConfigs(String expression, boolean allowNoMatch, Actio SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) .setSource(sourceBuilder) - .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) + .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 9840a142452b5..089ea178aa4ea 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -65,7 +65,6 @@ import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.MlStrings; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; @@ -92,7 +91,7 @@ * anomaly detector job configuration document * * The number of jobs returned in a search it limited to - * {@link AnomalyDetectorsIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}. + * {@link MlConfigIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}. * In most cases we expect 10s or 100s of jobs to be defined and * a search for all jobs should return all. */ @@ -521,7 +520,7 @@ public void expandJobsIds(String expression, SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) .setSource(sourceBuilder) - .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) + .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch); @@ -579,7 +578,7 @@ public void expandJobs(String expression, boolean allowNoMatch, boolean excludeD SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) .setSource(sourceBuilder) - .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) + .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch); @@ -637,7 +636,7 @@ public void expandGroupIds(List groupIds, ActionListener> listener) { SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) .setSource(sourceBuilder) - .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) + .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) .request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java index 59abf246bc762..fdcc790e325ae 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java @@ -83,7 +83,7 @@ public void testStateTemplate() { registry.clusterChanged(createClusterChangedEvent(nodes)); - verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject()); + verify(client.admin().indices(), times(4)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject()); PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream() .filter(r -> r.name().equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX)) @@ -99,7 +99,7 @@ public void testStatsTemplate() { registry.clusterChanged(createClusterChangedEvent(nodes)); - verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject()); + verify(client.admin().indices(), times(4)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject()); PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream() .filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME)) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java index f112bf52ea555..3a0eaec78f757 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction.TaskParams; -import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction.TaskExecutor; import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager; @@ -165,8 +164,7 @@ private static TaskExecutor createTaskExecutor() { mock(DataFrameAnalyticsManager.class), mock(DataFrameAnalyticsAuditor.class), mock(MlMemoryTracker.class), - new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - mock(IndexTemplateConfig.class)); + new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))); } private static DiscoveryNode createNode(int i, boolean isMlNode, Version nodeVersion) { diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/AbstractXPackRestTest.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/AbstractXPackRestTest.java index d3aa330bce212..7eadd098da7db 100644 --- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/AbstractXPackRestTest.java +++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/AbstractXPackRestTest.java @@ -91,10 +91,8 @@ private void waitForTemplates() throws Exception { templates.addAll( Arrays.asList( NotificationsIndex.NOTIFICATIONS_INDEX, - MlMetaIndex.indexName(), AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobResultsIndexPrefix(), - MlConfigIndex.indexName(), TransformInternalIndexConstants.AUDIT_INDEX, TransformInternalIndexConstants.LATEST_INDEX_NAME )); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java index c3c450d4f805b..8f446cc878a1a 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java @@ -44,7 +44,9 @@ protected Settings restClientSettings() { @Before public void waitForMlTemplates() throws Exception { - List templatesToWaitFor = XPackRestTestConstants.ML_POST_V660_TEMPLATES; + List templatesToWaitFor = getOldClusterVersion().onOrAfter(Version.V_7_12_0) + ? XPackRestTestConstants.ML_POST_V7120_TEMPLATES + : XPackRestTestConstants.ML_POST_V660_TEMPLATES; XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor); } diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java index 9dd26f5c82a4b..47fabf399a08a 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.restart; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; @@ -55,7 +56,9 @@ protected Settings restClientSettings() { @Before public void waitForMlTemplates() throws Exception { - List templatesToWaitFor = XPackRestTestConstants.ML_POST_V660_TEMPLATES; + List templatesToWaitFor = getOldClusterVersion().onOrAfter(Version.V_7_12_0) + ? XPackRestTestConstants.ML_POST_V7120_TEMPLATES + : XPackRestTestConstants.ML_POST_V660_TEMPLATES; XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java index 4245363eb8c8a..d43b7117c1382 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java @@ -74,7 +74,10 @@ private static class HLRC extends RestHighLevelClient { @Override protected Collection templatesToWaitFor() { - return Stream.concat(XPackRestTestConstants.ML_POST_V660_TEMPLATES.stream(), + List templatesToWaitFor = UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_12_0) + ? XPackRestTestConstants.ML_POST_V7120_TEMPLATES + : XPackRestTestConstants.ML_POST_V660_TEMPLATES; + return Stream.concat(templatesToWaitFor.stream(), super.templatesToWaitFor().stream()).collect(Collectors.toSet()); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index 00cfe0f4d4de5..b76868bb49b86 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -33,7 +34,10 @@ public class MlMappingsUpgradeIT extends AbstractUpgradeTestCase { @Override protected Collection templatesToWaitFor() { - return Stream.concat(XPackRestTestConstants.ML_POST_V660_TEMPLATES.stream(), + List templatesToWaitFor = UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_12_0) + ? XPackRestTestConstants.ML_POST_V7120_TEMPLATES + : XPackRestTestConstants.ML_POST_V660_TEMPLATES; + return Stream.concat(templatesToWaitFor.stream(), super.templatesToWaitFor().stream()).collect(Collectors.toSet()); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index c1123bfe84cf1..2930eb388db15 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -8,6 +8,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; import org.apache.lucene.util.TimeUnits; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; @@ -24,6 +25,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -37,7 +40,7 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa @Before public void waitForTemplates() throws Exception { try { - XPackRestTestHelper.waitForTemplates(client(), XPackRestTestConstants.ML_POST_V660_TEMPLATES); + XPackRestTestHelper.waitForTemplates(client(), XPackRestTestConstants.ML_POST_V7120_TEMPLATES); } catch (AssertionError e) { throw new AssertionError("Failure in test setup: Failed to initialize ML index templates", e); } diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java index 915055a17839a..3ca24dc106189 100644 --- a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java @@ -54,6 +54,30 @@ public class IndexMappingTemplateAsserter { * @throws IOException On error */ public static void assertMlMappingsMatchTemplates(RestClient client) throws IOException { + // Excluding those from stats index as some have been renamed and other removed. + // These exceptions are necessary for Full Cluster Restart tests where the upgrade version is < 7.x + Set statsIndexException = new HashSet<>(); + statsIndexException.add("properties.hyperparameters.properties.regularization_depth_penalty_multiplier.type"); + statsIndexException.add("properties.hyperparameters.properties.regularization_leaf_weight_penalty_multiplier.type"); + statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_limit.type"); + statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_tolerance.type"); + statsIndexException.add("properties.hyperparameters.properties.regularization_tree_size_penalty_multiplier.type"); + + // Excluding this from notifications index as `ignore_above` has been added to the `message.raw` field. + // The exception is necessary for Full Cluster Restart tests. + Set notificationsIndexExceptions = new HashSet<>(); + notificationsIndexExceptions.add("properties.message.fields.raw.ignore_above"); + + assertLegacyTemplateMatchesIndexMappings(client, ".ml-stats", ".ml-stats-000001", true, statsIndexException, false); + assertLegacyTemplateMatchesIndexMappings(client, ".ml-state", ".ml-state-000001", true, Collections.emptySet(), false); + // Depending on the order Full Cluster restart tests are run there may not be an notifications index yet + assertLegacyTemplateMatchesIndexMappings(client, + ".ml-notifications-000001", ".ml-notifications-000001", true, notificationsIndexExceptions, false); + // .ml-annotations-6 does not use a template + // .ml-anomalies-shared uses a template but will have dynamically updated mappings as new jobs are opened + + // TODO - add an endpoint to validate system index mappings - this will have to be done inside the cluster, not by REST calls + /* // Keys that have been dynamically mapped in the .ml-config index // but are not in the template. These can only be fixed with // re-index and should be addressed at the next major upgrade. @@ -69,36 +93,16 @@ public static void assertMlMappingsMatchTemplates(RestClient client) throws IOEx configIndexExceptions.add("properties.model_memory_limit.type"); // renamed to max_trees in 7.7. - // These exceptions are necessary for Full Cluster Restart tests where the upgrade version is < 7.x + // These exceptions are necessary for Full Cluster Restart tests where the upgrade version is < 7.7 configIndexExceptions.add("properties.analysis.properties.classification.properties.maximum_number_trees.type"); configIndexExceptions.add("properties.analysis.properties.regression.properties.maximum_number_trees.type"); - // Excluding those from stats index as some have been renamed and other removed. - // These exceptions are necessary for Full Cluster Restart tests where the upgrade version is < 7.x - Set statsIndexException = new HashSet<>(); - statsIndexException.add("properties.hyperparameters.properties.regularization_depth_penalty_multiplier.type"); - statsIndexException.add("properties.hyperparameters.properties.regularization_leaf_weight_penalty_multiplier.type"); - statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_limit.type"); - statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_tolerance.type"); - statsIndexException.add("properties.hyperparameters.properties.regularization_tree_size_penalty_multiplier.type"); - - // Excluding this from notifications index as `ignore_above` has been added to the `message.raw` field. - // The exception is necessary for Full Cluster Restart tests. - Set notificationsIndexExceptions = new HashSet<>(); - notificationsIndexExceptions.add("properties.message.fields.raw.ignore_above"); - assertLegacyTemplateMatchesIndexMappings(client, ".ml-config", ".ml-config", false, configIndexExceptions, true); // the true parameter means the index may not have been created assertLegacyTemplateMatchesIndexMappings(client, ".ml-meta", ".ml-meta", true, Collections.emptySet(), true); - assertLegacyTemplateMatchesIndexMappings(client, ".ml-stats", ".ml-stats-000001", true, statsIndexException, false); - assertLegacyTemplateMatchesIndexMappings(client, ".ml-state", ".ml-state-000001", true, Collections.emptySet(), false); - // Depending on the order Full Cluster restart tests are run there may not be an notifications index yet - assertLegacyTemplateMatchesIndexMappings(client, - ".ml-notifications-000001", ".ml-notifications-000001", true, notificationsIndexExceptions, false); assertLegacyTemplateMatchesIndexMappings(client, ".ml-inference-000003", ".ml-inference-000003", true, Collections.emptySet(), true); - // .ml-annotations-6 does not use a template - // .ml-anomalies-shared uses a template but will have dynamically updated mappings as new jobs are opened + */ } /** diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java index 64fa7ceb2f313..173d0531966b2 100644 --- a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java @@ -33,6 +33,11 @@ public final class XPackRestTestConstants { RESULTS_INDEX_PREFIX, CONFIG_INDEX); + public static final List ML_POST_V7120_TEMPLATES = + List.of( + STATE_INDEX_PREFIX, + RESULTS_INDEX_PREFIX); + // Transform constants: public static final String TRANSFORM_TASK_NAME = "data_frame/transforms"; public static final String TRANSFORM_INTERNAL_INDEX_PREFIX = ".transform-internal-"; From 827af64af4423325e61400b032c2fa40f1b599fe Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 27 Jan 2021 18:05:23 +0000 Subject: [PATCH 2/8] Fix problems shown by first round of CI --- .../restart/MlConfigIndexMappingsFullClusterRestartIT.java | 6 +++--- .../xpack/restart/MlMigrationFullClusterRestartIT.java | 6 +++--- .../upgrades/UpgradeClusterClientYamlTestSuiteIT.java | 3 --- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java index 8f446cc878a1a..29a2e30605a81 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java @@ -44,9 +44,9 @@ protected Settings restClientSettings() { @Before public void waitForMlTemplates() throws Exception { - List templatesToWaitFor = getOldClusterVersion().onOrAfter(Version.V_7_12_0) - ? XPackRestTestConstants.ML_POST_V7120_TEMPLATES - : XPackRestTestConstants.ML_POST_V660_TEMPLATES; + List templatesToWaitFor = (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_12_0)) + ? XPackRestTestConstants.ML_POST_V660_TEMPLATES + : XPackRestTestConstants.ML_POST_V7120_TEMPLATES; XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor); } diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java index 47fabf399a08a..1fee75d00b900 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java @@ -56,9 +56,9 @@ protected Settings restClientSettings() { @Before public void waitForMlTemplates() throws Exception { - List templatesToWaitFor = getOldClusterVersion().onOrAfter(Version.V_7_12_0) - ? XPackRestTestConstants.ML_POST_V7120_TEMPLATES - : XPackRestTestConstants.ML_POST_V660_TEMPLATES; + List templatesToWaitFor = (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_12_0)) + ? XPackRestTestConstants.ML_POST_V660_TEMPLATES + : XPackRestTestConstants.ML_POST_V7120_TEMPLATES; XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index 2930eb388db15..b4ffddbe3ca08 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -8,7 +8,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; import org.apache.lucene.util.TimeUnits; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; @@ -25,8 +24,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; From c6a03fb39af5b729af6fe8244a51d6d2dd3e7828 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 28 Jan 2021 13:30:10 +0000 Subject: [PATCH 3/8] Setting origin for mappings/settings updates from within the cluster --- .../core/ml/job/persistence/ElasticsearchMappings.java | 1 + .../org/elasticsearch/xpack/ml/MlConfigMigrator.java | 9 +-------- .../xpack/ml/support/BaseMlIntegTestCase.java | 6 +++++- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 1f1ac30336c77..2fe4cd7c61c17 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -162,6 +162,7 @@ public static void addDocMappingIfMissing(String alias, String mapping = mappingSupplier.get(); PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate); putMappingRequest.source(mapping, XContentType.JSON); + putMappingRequest.origin(ML_ORIGIN); executeAsyncWithOrigin(client, ML_ORIGIN, PutMappingAction.INSTANCE, putMappingRequest, ActionListener.wrap(response -> { if (response.isAcknowledged()) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index 3e65f51381819..beba0d4411b59 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -22,7 +22,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -33,7 +32,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.xpack.core.ml.MlConfigIndex; @@ -491,12 +489,7 @@ private void createConfigIndex(ActionListener listener) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(MlConfigIndex.indexName()); try { - createIndexRequest.settings( - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") - .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) - ); + createIndexRequest.settings(MlConfigIndex.settings()); createIndexRequest.mapping(MlConfigIndex.mapping()); } catch (Exception e) { logger.error("error writing the .ml-config mappings", e); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index ffa28ae16152a..e93fe2271ae87 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -40,6 +41,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.MockHttpTransport; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; @@ -468,7 +470,9 @@ protected String awaitJobOpenedAndAssigned(String jobId, String queryNode) throw * Sets delayed allocation to 0 to make sure we have tests are not delayed */ protected void setMlIndicesDelayedNodeLeftTimeoutToZero() { - client().admin().indices().updateSettings(new UpdateSettingsRequest(".ml-*") + OriginSettingClient originSettingClient = new OriginSettingClient(client(), ClientHelper.ML_ORIGIN); + originSettingClient.admin().indices().updateSettings(new UpdateSettingsRequest(".ml-*") + .origin(ClientHelper.ML_ORIGIN) .settings(Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0).build())) .actionGet(); } From 767439ce749da68fe703d8c0520ed185f10b8f0d Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 2 Feb 2021 19:21:18 +0000 Subject: [PATCH 4/8] System indices don't work with dynamic mappings So we have to make sure all mappings are created upfront. For fields we originally missed we have to stick with the type that would have been chosen dynamically for users who acquired dynamic mappings in previous versions. --- .../put/TransportPutMappingAction.java | 2 +- .../xpack/core/ml/config_index_mappings.json | 36 +++++++++++++++---- .../xpack/core/ml/meta_index_mappings.json | 9 +++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 2c1f99c4c0af3..cea348e8d49de 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -183,7 +183,7 @@ static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] return "Cannot update mappings in " + violations + ": system indices can only use mappings from their descriptors, " - + "but the mappings in the request did not match those in the descriptors(s)"; + + "but the mappings in the request [" + requestMappings + "] did not match those in the descriptor(s)"; } return null; diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json index 01e0439a5bb23..f01e1d9a61abc 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json @@ -21,6 +21,9 @@ "allow_lazy_open" : { "type" : "keyword" }, + "allow_lazy_start" : { + "type" : "keyword" + }, "analysis" : { "properties" : { "classification" : { @@ -34,6 +37,9 @@ "downsample_factor" : { "type" : "double" }, + "early_stopping_enabled" : { + "type": "boolean" + }, "eta" : { "type" : "double" }, @@ -70,6 +76,9 @@ "prediction_field_name" : { "type" : "keyword" }, + "randomize_seed" : { + "type" : "keyword" + }, "soft_tree_depth_limit" : { "type" : "double" }, @@ -78,22 +87,28 @@ }, "training_percent" : { "type" : "double" - }, - "early_stopping_enabled" : { - "type": "boolean" } } }, "outlier_detection" : { "properties" : { + "compute_feature_influence" : { + "type" : "keyword" + }, "feature_influence_threshold" : { "type" : "double" }, "method" : { "type" : "keyword" }, + "outlier_fraction" : { + "type" : "keyword" + }, "n_neighbors" : { "type" : "integer" + }, + "standardization_enabled" : { + "type" : "keyword" } } }, @@ -108,6 +123,9 @@ "downsample_factor" : { "type" : "double" }, + "early_stopping_enabled" : { + "type": "boolean" + }, "eta" : { "type" : "double" }, @@ -144,6 +162,9 @@ "prediction_field_name" : { "type" : "keyword" }, + "randomize_seed" : { + "type" : "keyword" + }, "soft_tree_depth_limit" : { "type" : "double" }, @@ -152,9 +173,6 @@ }, "training_percent" : { "type" : "double" - }, - "early_stopping_enabled" : { - "type": "boolean" } } } @@ -328,6 +346,9 @@ } } }, + "deleting" : { + "type" : "keyword" + }, "description" : { "type" : "text" }, @@ -373,6 +394,9 @@ "job_version" : { "type" : "keyword" }, + "max_empty_searches" : { + "type" : "keyword" + }, "max_num_threads" : { "type" : "integer" }, diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json index 4f038650f9ad7..f78414f53ea4f 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json @@ -23,11 +23,20 @@ "description": { "type": "keyword" }, + "filter_id": { + "type": "keyword" + }, + "items": { + "type": "keyword" + }, "start_time": { "type": "date" }, "end_time": { "type": "date" + }, + "type": { + "type": "keyword" } } } From d6b119435dac24e89bcfa71fd3cd4f138c71984d Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 3 Feb 2021 11:17:17 +0000 Subject: [PATCH 5/8] Reserved fields for the config index are unnecessary We have the concept of reserved fields for our results indices because end user fields get added to results and we need to ensure they don't clash with fields we want to use ourselves. This problem does not exist for the config index, as we don't add arbitrary end user fields. Therefore we don't need reserved fields for the config index. --- .../ml/job/results/ReservedFieldNames.java | 165 ------------------ .../ElasticsearchMappingsTests.java | 41 +---- 2 files changed, 8 insertions(+), 198 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 2f5679a726ea2..43d017d68dd12 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -7,27 +7,9 @@ package org.elasticsearch.xpack.core.ml.job.results; import org.elasticsearch.index.get.GetResult; -import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; -import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig; -import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; -import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; -import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource; -import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams; -import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification; -import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection; -import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; -import org.elasticsearch.xpack.core.ml.job.config.DataDescription; -import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; -import org.elasticsearch.xpack.core.ml.job.config.Operator; -import org.elasticsearch.xpack.core.ml.job.config.PerPartitionCategorizationConfig; -import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; @@ -214,148 +196,6 @@ public final class ReservedFieldNames { GetResult._ID, GetResult._INDEX - }; - - /** - * This array should be updated to contain all the field names that appear - * in any documents we store in our config index. - */ - private static final String[] RESERVED_CONFIG_FIELD_NAME_ARRAY = { - Job.ID.getPreferredName(), - Job.JOB_TYPE.getPreferredName(), - Job.JOB_VERSION.getPreferredName(), - Job.GROUPS.getPreferredName(), - Job.ANALYSIS_CONFIG.getPreferredName(), - Job.ANALYSIS_LIMITS.getPreferredName(), - Job.CREATE_TIME.getPreferredName(), - Job.CUSTOM_SETTINGS.getPreferredName(), - Job.DATA_DESCRIPTION.getPreferredName(), - Job.DESCRIPTION.getPreferredName(), - Job.FINISHED_TIME.getPreferredName(), - Job.MODEL_PLOT_CONFIG.getPreferredName(), - Job.RENORMALIZATION_WINDOW_DAYS.getPreferredName(), - Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName(), - Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), - Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), - Job.RESULTS_RETENTION_DAYS.getPreferredName(), - Job.MODEL_SNAPSHOT_ID.getPreferredName(), - Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), - Job.RESULTS_INDEX_NAME.getPreferredName(), - Job.ALLOW_LAZY_OPEN.getPreferredName(), - - AnalysisConfig.BUCKET_SPAN.getPreferredName(), - AnalysisConfig.CATEGORIZATION_FIELD_NAME.getPreferredName(), - AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), - AnalysisConfig.CATEGORIZATION_ANALYZER.getPreferredName(), - AnalysisConfig.PER_PARTITION_CATEGORIZATION.getPreferredName(), - AnalysisConfig.LATENCY.getPreferredName(), - AnalysisConfig.SUMMARY_COUNT_FIELD_NAME.getPreferredName(), - AnalysisConfig.DETECTORS.getPreferredName(), - AnalysisConfig.INFLUENCERS.getPreferredName(), - AnalysisConfig.MULTIVARIATE_BY_FIELDS.getPreferredName(), - - AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName(), - AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), - - Detector.DETECTOR_DESCRIPTION_FIELD.getPreferredName(), - Detector.FUNCTION_FIELD.getPreferredName(), - Detector.FIELD_NAME_FIELD.getPreferredName(), - Detector.BY_FIELD_NAME_FIELD.getPreferredName(), - Detector.OVER_FIELD_NAME_FIELD.getPreferredName(), - Detector.PARTITION_FIELD_NAME_FIELD.getPreferredName(), - Detector.USE_NULL_FIELD.getPreferredName(), - Detector.EXCLUDE_FREQUENT_FIELD.getPreferredName(), - Detector.CUSTOM_RULES_FIELD.getPreferredName(), - Detector.DETECTOR_INDEX.getPreferredName(), - - DetectionRule.ACTIONS_FIELD.getPreferredName(), - DetectionRule.CONDITIONS_FIELD.getPreferredName(), - DetectionRule.SCOPE_FIELD.getPreferredName(), - RuleCondition.APPLIES_TO_FIELD.getPreferredName(), - RuleCondition.VALUE_FIELD.getPreferredName(), - Operator.OPERATOR_FIELD.getPreferredName(), - - DataDescription.FORMAT_FIELD.getPreferredName(), - DataDescription.TIME_FIELD_NAME_FIELD.getPreferredName(), - DataDescription.TIME_FORMAT_FIELD.getPreferredName(), - DataDescription.FIELD_DELIMITER_FIELD.getPreferredName(), - DataDescription.QUOTE_CHARACTER_FIELD.getPreferredName(), - - ModelPlotConfig.ENABLED_FIELD.getPreferredName(), - ModelPlotConfig.TERMS_FIELD.getPreferredName(), - ModelPlotConfig.ANNOTATIONS_ENABLED_FIELD.getPreferredName(), - - PerPartitionCategorizationConfig.STOP_ON_WARN.getPreferredName(), - - DatafeedConfig.ID.getPreferredName(), - DatafeedConfig.QUERY_DELAY.getPreferredName(), - DatafeedConfig.FREQUENCY.getPreferredName(), - DatafeedConfig.INDICES.getPreferredName(), - DatafeedConfig.QUERY.getPreferredName(), - DatafeedConfig.SCROLL_SIZE.getPreferredName(), - DatafeedConfig.AGGREGATIONS.getPreferredName(), - DatafeedConfig.SCRIPT_FIELDS.getPreferredName(), - DatafeedConfig.CHUNKING_CONFIG.getPreferredName(), - DatafeedConfig.HEADERS.getPreferredName(), - DatafeedConfig.DELAYED_DATA_CHECK_CONFIG.getPreferredName(), - DatafeedConfig.INDICES_OPTIONS.getPreferredName(), - DelayedDataCheckConfig.ENABLED.getPreferredName(), - DelayedDataCheckConfig.CHECK_WINDOW.getPreferredName(), - - ChunkingConfig.MODE_FIELD.getPreferredName(), - ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(), - - DataFrameAnalyticsConfig.ID.getPreferredName(), - DataFrameAnalyticsConfig.DESCRIPTION.getPreferredName(), - DataFrameAnalyticsConfig.SOURCE.getPreferredName(), - DataFrameAnalyticsConfig.DEST.getPreferredName(), - DataFrameAnalyticsConfig.ANALYSIS.getPreferredName(), - DataFrameAnalyticsConfig.ANALYZED_FIELDS.getPreferredName(), - DataFrameAnalyticsConfig.CREATE_TIME.getPreferredName(), - DataFrameAnalyticsConfig.VERSION.getPreferredName(), - DataFrameAnalyticsConfig.MAX_NUM_THREADS.getPreferredName(), - DataFrameAnalyticsDest.INDEX.getPreferredName(), - DataFrameAnalyticsDest.RESULTS_FIELD.getPreferredName(), - DataFrameAnalyticsSource.INDEX.getPreferredName(), - DataFrameAnalyticsSource.QUERY.getPreferredName(), - DataFrameAnalyticsSource._SOURCE.getPreferredName(), - OutlierDetection.NAME.getPreferredName(), - OutlierDetection.N_NEIGHBORS.getPreferredName(), - OutlierDetection.METHOD.getPreferredName(), - OutlierDetection.FEATURE_INFLUENCE_THRESHOLD.getPreferredName(), - Regression.NAME.getPreferredName(), - Regression.DEPENDENT_VARIABLE.getPreferredName(), - Regression.LOSS_FUNCTION.getPreferredName(), - Regression.LOSS_FUNCTION_PARAMETER.getPreferredName(), - Regression.PREDICTION_FIELD_NAME.getPreferredName(), - Regression.TRAINING_PERCENT.getPreferredName(), - Regression.FEATURE_PROCESSORS.getPreferredName(), - Regression.EARLY_STOPPING_ENABLED.getPreferredName(), - Classification.NAME.getPreferredName(), - Classification.DEPENDENT_VARIABLE.getPreferredName(), - Classification.PREDICTION_FIELD_NAME.getPreferredName(), - Classification.CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), - Classification.NUM_TOP_CLASSES.getPreferredName(), - Classification.TRAINING_PERCENT.getPreferredName(), - Classification.FEATURE_PROCESSORS.getPreferredName(), - Classification.EARLY_STOPPING_ENABLED.getPreferredName(), - BoostedTreeParams.ALPHA.getPreferredName(), - BoostedTreeParams.DOWNSAMPLE_FACTOR.getPreferredName(), - BoostedTreeParams.LAMBDA.getPreferredName(), - BoostedTreeParams.GAMMA.getPreferredName(), - BoostedTreeParams.ETA.getPreferredName(), - BoostedTreeParams.ETA_GROWTH_RATE_PER_TREE.getPreferredName(), - BoostedTreeParams.MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER.getPreferredName(), - BoostedTreeParams.MAX_TREES.getPreferredName(), - BoostedTreeParams.FEATURE_BAG_FRACTION.getPreferredName(), - BoostedTreeParams.NUM_TOP_FEATURE_IMPORTANCE_VALUES.getPreferredName(), - BoostedTreeParams.SOFT_TREE_DEPTH_LIMIT.getPreferredName(), - BoostedTreeParams.SOFT_TREE_DEPTH_TOLERANCE.getPreferredName(), - - ElasticsearchMappings.CONFIG_TYPE, - - GetResult._ID, - GetResult._INDEX, }; /** @@ -379,11 +219,6 @@ public static boolean isValidFieldName(String fieldName) { */ public static final Set RESERVED_RESULT_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_RESULT_FIELD_NAME_ARRAY)); - /** - * A set of all reserved field names in our config. - */ - public static final Set RESERVED_CONFIG_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_CONFIG_FIELD_NAME_ARRAY)); - private ReservedFieldNames() { } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java index 82a2739dd8501..5af9f1b93f335 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java @@ -28,11 +28,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.MlConfigIndex; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; -import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; @@ -109,22 +105,6 @@ public void testResultsMappingReservedFields() throws Exception { compareFields(expected, ReservedFieldNames.RESERVED_RESULT_FIELD_NAMES); } - public void testConfigMappingReservedFields() throws Exception { - Set overridden = new HashSet<>(KEYWORDS); - - // These are not reserved because they're data types, not field names - overridden.add(Job.TYPE); - overridden.add(DatafeedConfig.TYPE); - // ModelPlotConfig has an 'enabled' the same as one of the keywords - overridden.remove(ModelPlotConfig.ENABLED_FIELD.getPreferredName()); - - Set expected = collectConfigDocFieldNames(); - expected.removeAll(overridden); - expected.addAll(INTERNAL_FIELDS); - - compareFields(expected, ReservedFieldNames.RESERVED_CONFIG_FIELD_NAMES); - } - private void compareFields(Set expected, Set reserved) { if (reserved.size() != expected.size()) { Set diff = new HashSet<>(reserved); @@ -145,7 +125,7 @@ private void compareFields(Set expected, Set reserved) { } } - public void testMappingRequiresUpdateNoMapping() throws IOException { + public void testMappingRequiresUpdateNoMapping() { ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); ClusterState cs = csBuilder.build(); String[] indices = new String[] { "no_index" }; @@ -153,44 +133,44 @@ public void testMappingRequiresUpdateNoMapping() throws IOException { assertArrayEquals(new String[] { "no_index" }, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); } - public void testMappingRequiresUpdateNullMapping() throws IOException { + public void testMappingRequiresUpdateNullMapping() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("null_mapping", null)); String[] indices = new String[] { "null_index" }; assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); } - public void testMappingRequiresUpdateNoVersion() throws IOException { + public void testMappingRequiresUpdateNoVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("no_version_field", "NO_VERSION_FIELD")); String[] indices = new String[] { "no_version_field" }; assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); } - public void testMappingRequiresUpdateRecentMappingVersion() throws IOException { + public void testMappingRequiresUpdateRecentMappingVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_current", Version.CURRENT.toString())); String[] indices = new String[] { "version_current" }; assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); } - public void testMappingRequiresUpdateMaliciousMappingVersion() throws IOException { + public void testMappingRequiresUpdateMaliciousMappingVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata( Collections.singletonMap("version_current", Collections.singletonMap("nested", "1.0"))); String[] indices = new String[] { "version_nested" }; assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); } - public void testMappingRequiresUpdateBogusMappingVersion() throws IOException { + public void testMappingRequiresUpdateBogusMappingVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_bogus", "0.0")); String[] indices = new String[] { "version_bogus" }; assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); } - public void testMappingRequiresUpdateNewerMappingVersion() throws IOException { + public void testMappingRequiresUpdateNewerMappingVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", Version.CURRENT)); String[] indices = new String[] { "version_newer" }; assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion())); } - public void testMappingRequiresUpdateNewerMappingVersionMinor() throws IOException { + public void testMappingRequiresUpdateNewerMappingVersionMinor() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", Version.CURRENT)); String[] indices = new String[] { "version_newer_minor" }; assertArrayEquals(new String[] {}, @@ -274,11 +254,6 @@ private Set collectResultsDocFieldNames() throws IOException { return collectFieldNames(AnomalyDetectorsIndex.resultsMapping()); } - private Set collectConfigDocFieldNames() throws IOException { - // Only the mappings for the config index should be added below. Do NOT add mappings for other indexes here. - return collectFieldNames(MlConfigIndex.mapping()); - } - private Set collectFieldNames(String mapping) throws IOException { BufferedInputStream inputStream = new BufferedInputStream(new ByteArrayInputStream(mapping.getBytes(StandardCharsets.UTF_8))); From 04d22e294672ac9545d5560293489a686d9aa2d9 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 3 Feb 2021 11:29:31 +0000 Subject: [PATCH 6/8] Dynamic updates are banned for system indices This avoids the risk of future incorrect mappings caused by dynamic updates --- .../rest/IndexMappingTemplateAsserter.java | 34 ++++--------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java index 8957e1ebb513c..77b3e1203f7de 100644 --- a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java @@ -77,33 +77,13 @@ public static void assertMlMappingsMatchTemplates(RestClient client) throws IOEx // .ml-annotations-6 does not use a template // .ml-anomalies-shared uses a template but will have dynamically updated mappings as new jobs are opened - // TODO - add an endpoint to validate system index mappings - this will have to be done inside the cluster, not by REST calls - /* - // Keys that have been dynamically mapped in the .ml-config index - // but are not in the template. These can only be fixed with - // re-index and should be addressed at the next major upgrade. - // For now this serves as documentation of the missing fields - Set configIndexExceptions = new HashSet<>(); - configIndexExceptions.add("properties.allow_lazy_start.type"); - configIndexExceptions.add("properties.analysis.properties.classification.properties.randomize_seed.type"); - configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.compute_feature_influence.type"); - configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.outlier_fraction.type"); - configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.standardization_enabled.type"); - configIndexExceptions.add("properties.analysis.properties.regression.properties.randomize_seed.type"); - configIndexExceptions.add("properties.deleting.type"); - configIndexExceptions.add("properties.model_memory_limit.type"); - - // renamed to max_trees in 7.7. - // These exceptions are necessary for Full Cluster Restart tests where the upgrade version is < 7.7 - configIndexExceptions.add("properties.analysis.properties.classification.properties.maximum_number_trees.type"); - configIndexExceptions.add("properties.analysis.properties.regression.properties.maximum_number_trees.type"); - - assertLegacyTemplateMatchesIndexMappings(client, ".ml-config", ".ml-config", false, configIndexExceptions, true); - // the true parameter means the index may not have been created - assertLegacyTemplateMatchesIndexMappings(client, ".ml-meta", ".ml-meta", true, Collections.emptySet(), true); - assertLegacyTemplateMatchesIndexMappings(client, - ".ml-inference-000003", ".ml-inference-000003", true, Collections.emptySet(), true); - */ + // Dynamic mappings updates are banned for system indices. + // The .ml-config and .ml-meta indices have mappings that allow dynamic updates. + // The effect is instant error if a document containing an unknown field is added + // to one of these indices. Assuming we have some sort of test coverage somewhere + // for new fields, we will very quickly catch any failures to add new fields to + // the mappings for the .ml-config and .ml-meta indices. So there is no need to + // test again here. } /** From d05edb934af9933c7afb1f93ecf5ee71ec57f367 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 3 Feb 2021 18:51:03 +0000 Subject: [PATCH 7/8] Ensure inference index is created with correct mappings in mixed version clusters --- .../xpack/core/ml/utils/MlIndexAndAlias.java | 48 +++++++++++++++++++ .../xpack/ml/MachineLearning.java | 22 +++++---- .../xpack/ml/MlConfigMigrator.java | 1 + ...ransportStartDataFrameAnalyticsAction.java | 24 +++++++++- 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index 13cc33111a3a5..9c6d5cd1984ac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -30,6 +30,8 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import java.util.Arrays; @@ -162,6 +164,48 @@ public static void createIndexAndAliasIfNecessary(Client client, loggingListener.onResponse(false); } + public static void createSystemIndexIfNecessary(Client client, + ClusterState clusterState, + SystemIndexDescriptor descriptor, + ActionListener finalListener) { + + final String primaryIndex = descriptor.getPrimaryIndex(); + + // The check for existence of the index is against the cluster state, so very cheap + if (hasIndex(clusterState, primaryIndex)) { + finalListener.onResponse(true); + return; + } + + ActionListener indexCreatedListener = ActionListener.wrap( + created -> { + if (created) { + waitForShardsReady(client, primaryIndex, finalListener); + } else { + finalListener.onResponse(false); + } + }, + e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) { + finalListener.onResponse(true); + } else { + finalListener.onFailure(e); + } + } + ); + + CreateIndexRequest createIndexRequest = new CreateIndexRequest(primaryIndex); + createIndexRequest.settings(descriptor.getSettings()); + createIndexRequest.mapping(descriptor.getMappings()); + createIndexRequest.origin(ML_ORIGIN); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest, + ActionListener.wrap( + r -> indexCreatedListener.onResponse(r.isAcknowledged()), + indexCreatedListener::onFailure + ), client.admin().indices()::create); + } + private static void waitForShardsReady(Client client, String index, ActionListener listener) { ClusterHealthRequest healthRequest = Requests.clusterHealthRequest(index) .waitForYellowStatus() @@ -309,4 +353,8 @@ public static void installIndexTemplateIfRequired( public static boolean hasIndexTemplate(ClusterState state, String templateName) { return state.getMetadata().getTemplates().containsKey(templateName); } + + public static boolean hasIndex(ClusterState state, String index) { + return state.getMetadata().getIndicesLookup().containsKey(index); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 7a314fee74846..ab869dec02387 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1180,18 +1180,22 @@ public Collection getSystemIndexDescriptors(Settings sett .setVersionMetaKey("version") .setOrigin(ML_ORIGIN) .build(), - SystemIndexDescriptor.builder() - .setIndexPattern(InferenceIndexConstants.INDEX_PATTERN) - .setPrimaryIndex(InferenceIndexConstants.LATEST_INDEX_NAME) - .setDescription("Contains ML model configuration and statistics") - .setMappings(InferenceIndexConstants.mapping()) - .setSettings(InferenceIndexConstants.settings()) - .setVersionMetaKey("version") - .setOrigin(ML_ORIGIN) - .build() + getInferenceIndexSecurityDescriptor() ); } + public static SystemIndexDescriptor getInferenceIndexSecurityDescriptor() { + return SystemIndexDescriptor.builder() + .setIndexPattern(InferenceIndexConstants.INDEX_PATTERN) + .setPrimaryIndex(InferenceIndexConstants.LATEST_INDEX_NAME) + .setDescription("Contains ML model configuration and statistics") + .setMappings(InferenceIndexConstants.mapping()) + .setSettings(InferenceIndexConstants.settings()) + .setVersionMetaKey("version") + .setOrigin(ML_ORIGIN) + .build(); + } + @Override public BreakerSettings getCircuitBreaker(Settings settings) { return BreakerSettings.updateFromSettings( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index a50ed4cf6c0eb..8bef0cdabf497 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -492,6 +492,7 @@ private void createConfigIndex(ActionListener listener) { { createIndexRequest.settings(MlConfigIndex.settings()); createIndexRequest.mapping(MlConfigIndex.mapping()); + createIndexRequest.origin(ML_ORIGIN); } catch (Exception e) { logger.error("error writing the .ml-config mappings", e); listener.onFailure(e); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index 985652d0fc17a..a5999f80a3954 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -61,9 +61,11 @@ import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState; import org.elasticsearch.xpack.core.ml.dataframe.analyses.RequiredField; +import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.core.ml.utils.PhaseProgress; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager; @@ -669,8 +671,26 @@ protected void nodeOperation(AllocatedPersistentTask task, TaskParams params, Pe ); // Get stats to initialize in memory stats tracking - executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsStatsAction.INSTANCE, - new GetDataFrameAnalyticsStatsAction.Request(params.getId()), statsListener); + ActionListener indexCheckListener = ActionListener.wrap( + ok -> executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsStatsAction.INSTANCE, + new GetDataFrameAnalyticsStatsAction.Request(params.getId()), statsListener), + error -> { + Throwable cause = ExceptionsHelper.unwrapCause(error); + logger.error( + new ParameterizedMessage( + "[{}] failed to create internal index [{}]", + params.getId(), + InferenceIndexConstants.LATEST_INDEX_NAME), + cause); + dfaTask.setFailed(error); + } + ); + + // Create the system index explicitly. Although the master node would create it automatically on first use, + // in a mixed version cluster where the master node is on an older version than this node relying on auto-creation + // might use outdated mappings. + MlIndexAndAlias.createSystemIndexIfNecessary(client, clusterState, MachineLearning.getInferenceIndexSecurityDescriptor(), + indexCheckListener); } private void executeTask(DataFrameAnalyticsTask task) { From e3df1b3a8be2d29fd8ab394813eb5e3ebf103be1 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 3 Feb 2021 19:30:41 +0000 Subject: [PATCH 8/8] Remove unused import --- .../org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index 9c6d5cd1984ac..5c98904095740 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.indices.SystemIndexDescriptor; -import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import java.util.Arrays;