diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 5d5faacc10a2a..6feff5a6b11a2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -256,12 +256,13 @@ public Iterator> settings() { Setting.Property.Dynamic, Setting.Property.IndexScope); + public static final String SETTING_INDEX_HIDDEN = "index.hidden"; /** * Whether the index is considered hidden or not. A hidden index will not be resolved in * normal wildcard searches unless explicitly allowed */ public static final Setting INDEX_HIDDEN_SETTING = - Setting.boolSetting("index.hidden", false, Property.IndexScope, Property.Final); + Setting.boolSetting(SETTING_INDEX_HIDDEN, false, Property.IndexScope, Property.Final); /** * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index abfcdf7af5e2c..9bac4976be212 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -119,9 +119,6 @@ public class MetaDataCreateIndexService { */ private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton( ".watch-history-*", - ".ml-anomalies-*", - ".ml-notifications-*", - ".ml-annotations*", ".data-frame-notifications-*", ".transform-notifications-*" )); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 9a955146d18df..7d0d065f611ff 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -618,9 +618,6 @@ public void testIndexNameExclusionsList() { // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty List excludedNames = Arrays.asList( ".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), - ".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), - ".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), - ".ml-annotations-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".data-frame-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".transform-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT) ); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 79ded94250da6..ffef57ddf867a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterName; @@ -124,13 +125,20 @@ public void setUp() throws Exception { public void tearDown() throws Exception { logger.trace("[{}#{}]: cleaning up after test", getTestClass().getSimpleName(), getTestName()); super.tearDown(); - assertAcked(client().admin().indices().prepareDelete("*").get()); + assertAcked( + client().admin().indices().prepareDelete("*") + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .get()); MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData(); assertThat("test leaves persistent cluster metadata behind: " + metaData.persistentSettings().keySet(), metaData.persistentSettings().size(), equalTo(0)); assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().keySet(), metaData.transientSettings().size(), equalTo(0)); - GetIndexResponse indices = client().admin().indices().prepareGetIndex().addIndices("*").get(); + GetIndexResponse indices = + client().admin().indices().prepareGetIndex() + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .addIndices("*") + .get(); assertThat("test leaves indices that were not deleted: " + Strings.arrayToCommaDelimitedString(indices.indices()), indices.indices(), equalTo(Strings.EMPTY_ARRAY)); if (resetNodeAfterTest()) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 6947d09bb699a..2ecac6daac8f4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -572,9 +572,9 @@ private void wipeCluster() throws Exception { protected static void wipeAllIndices() throws IOException { boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0); try { - final Request deleteReq = new Request("DELETE", "*"); - deleteReq.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : "")); - final Response response = adminClient().performRequest(deleteReq); + final Request deleteRequest = new Request("DELETE", "*"); + deleteRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : "")); + final Response response = adminClient().performRequest(deleteRequest); try (InputStream is = response.getEntity().getContent()) { assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged")); } @@ -706,6 +706,13 @@ private void wipeRollupJobs() throws IOException { } } + protected void refreshAllIndices() throws IOException { + boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0); + Request refreshRequest = new Request("POST", "/_refresh"); + refreshRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : "")); + client().performRequest(refreshRequest); + } + private void waitForPendingRollupTasks() throws Exception { waitForPendingTasks(adminClient(), taskName -> taskName.startsWith("xpack/rollup/job") == false); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 705c06a70e1a4..2ef467562929b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -59,11 +59,13 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c // Create the annotations index if it doesn't exist already. if (mlLookup.containsKey(INDEX_NAME) == false) { - CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME); - createIndexRequest.mapping(annotationsMapping()); - createIndexRequest.settings(Settings.builder() - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")); + CreateIndexRequest createIndexRequest = + new CreateIndexRequest(INDEX_NAME) + .mapping(annotationsMapping()) + .settings(Settings.builder() + .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexMetaData.SETTING_INDEX_HIDDEN, true)); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest, ActionListener.wrap( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java index abcfc19efbf80..5fad414c879e1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java @@ -9,11 +9,9 @@ public final class AnomalyDetectorsIndexFields { public static final String CONFIG_INDEX = ".ml-config"; - public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-"; - public static final String STATE_INDEX_PREFIX = ".ml-state"; - public static final String STATE_INDEX_PATTERN = STATE_INDEX_PREFIX + "*"; + public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-"; public static final String RESULTS_INDEX_DEFAULT = "shared"; private AnomalyDetectorsIndexFields() {} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json index 37a759cba7325..bccad3e753678 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json @@ -12,7 +12,8 @@ "auto_expand_replicas" : "0-1", "query" : { "default_field" : "all_field_values" - } + }, + "hidden": true } }, "mappings": ${xpack.ml.anomalydetection.results.mappings} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json index 39211ae5b20ff..fb0b7ad6c2bd5 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json @@ -6,7 +6,8 @@ ], "settings" : { "index" : { - "auto_expand_replicas" : "0-1" + "auto_expand_replicas" : "0-1", + "hidden": true } }, "mappings" : { diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json index 804707dfb189f..e8ac3f1fcce6f 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json @@ -7,7 +7,8 @@ "settings" : { "index" : { "number_of_shards" : "1", - "auto_expand_replicas" : "0-1" + "auto_expand_replicas" : "0-1", + "hidden": true } }, "mappings" : { diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/test/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/test/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index 14be41bbf269d..c2070fb2effed 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/test/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/test/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -127,7 +127,7 @@ public void testMiniFarequoteWithDatafeeder() throws Exception { client().performRequest(airlineData2); // Ensure all data is searchable - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); String jobId = "mini-farequote-with-data-feeder-job"; createFarequoteJob(jobId); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java index 36d0b141093ee..4ca8fb5a64373 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -79,7 +80,7 @@ public void setUpData() { public void tearDownData() { cleanUp(); client().admin().indices().prepareDelete(DATA_INDEX).get(); - client().admin().indices().prepareRefresh("*").get(); + client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get(); } public void testBasicCategorization() throws Exception { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index 3e6837affb046..f5e229d572bcd 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -747,7 +747,7 @@ public void testLookbackWithoutPermissions() throws Exception { assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":0")); // There should be a notification saying that there was a problem extracting data - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); Response notificationsResponse = client().performRequest( new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId)); String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity()); @@ -954,7 +954,7 @@ public void testLookbackWithoutPermissionsAndRollup() throws Exception { startDatafeedAndWaitUntilStopped(datafeedId, BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS); waitUntilJobIsClosed(jobId); // There should be a notification saying that there was a problem extracting data - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); Response notificationsResponse = client().performRequest( new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId)); String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity()); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index 381ab7651ac70..7d1506286362b 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequest; @@ -161,7 +162,7 @@ public void testDeleteExpiredData() throws Exception { } // Refresh to ensure the snapshot timestamp updates are visible - client().admin().indices().prepareRefresh("*").get(); + client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get(); // We need to wait a second to ensure the second time around model snapshots will have a different ID (it depends on epoch seconds) // FIXME it would be better to wait for something concrete instead of wait for time to elapse @@ -292,6 +293,6 @@ private void retainAllSnapshots(String jobId) throws Exception { client().execute(UpdateModelSnapshotAction.INSTANCE, request).get(); } // We need to refresh to ensure the updates are visible - client().admin().indices().prepareRefresh("*").get(); + client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get(); } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 68669b0dbc8ef..bc22667ef9418 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -231,7 +231,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { jobId1, "1236", 1)); client().performRequest(createResultRequest); - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); responseAsString = EntityUtils.toString(client().performRequest( new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1 + "/results/buckets")).getEntity()); @@ -256,7 +256,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { jobId2, "1236", 1)); client().performRequest(createResultRequest); - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); responseAsString = EntityUtils.toString(client().performRequest( new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2 + "/results/buckets")).getEntity()); @@ -278,7 +278,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); responseAsString = EntityUtils.toString(client().performRequest( new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "/_count")).getEntity()); @@ -289,7 +289,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity()); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); responseAsString = EntityUtils.toString(client().performRequest( new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName))); @@ -670,7 +670,7 @@ public void testMultiIndexDelete() throws Exception { createDoc3.setEntity(createDoc0.getEntity()); client().performRequest(createDoc3); - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); // check for the documents assertThat(EntityUtils.toString(client().performRequest(new Request("GET", indexName+ "/_count")).getEntity()), @@ -683,7 +683,7 @@ public void testMultiIndexDelete() throws Exception { // Delete client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId)); - client().performRequest(new Request("POST", "/_refresh")); + refreshAllIndices(); // check that the indices still exist but are empty String indicesAfterDelete = EntityUtils.toString(client().performRequest( diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 2393ad1023549..d8cba1c772589 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterState; @@ -148,7 +149,7 @@ protected DeleteExpiredDataAction.Response deleteExpiredData() throws Exception new DeleteExpiredDataAction.Request()).get(); // We need to refresh to ensure the deletion is visible - client().admin().indices().prepareRefresh("*").get(); + client().admin().indices().prepareRefresh("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get(); return response; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 2314ec420482c..fa4b0ef60bd7d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -938,7 +938,6 @@ public List getNamedXContent() { public Collection getSystemIndexDescriptors() { return List.of( new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, "Contains scheduling and anomaly tracking metadata"), - new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, "Contains ML model state"), new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, "Contains ML configuration data"), new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, "Contains ML model configuration and statistics") ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java index 2b7079a97f0a0..f5d3fc84b8eeb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml.integration; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -73,7 +74,10 @@ private boolean annotationsIndexExists() { private int numberOfAnnotationsAliases() { int count = 0; ImmutableOpenMap> aliases = client().admin().indices() - .prepareGetAliases(AnnotationIndex.READ_ALIAS_NAME, AnnotationIndex.WRITE_ALIAS_NAME).get().getAliases(); + .prepareGetAliases(AnnotationIndex.READ_ALIAS_NAME, AnnotationIndex.WRITE_ALIAS_NAME) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .get() + .getAliases(); if (aliases != null) { for (ObjectObjectCursor> entry : aliases) { count += entry.value.size(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index 93b7eb06b3efd..899be32a217cb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -165,7 +165,7 @@ public void testJobAutoClose() throws Exception { indexRequest = new IndexRequest("data"); indexRequest.source("time", 1407083600L); client().index(indexRequest).get(); - refresh(); + refresh("*", ".ml-*"); Job.Builder job = createScheduledJob("job_id"); PutJobAction.Request putJobRequest = new PutJobAction.Request(job); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index 2ff8f465af156..84a26b9661a2c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -7,6 +7,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -489,6 +490,7 @@ private void run(String jobId, CheckedRunnable disrupt) throws Except // are what we expect them to be: private static DataCounts getDataCountsFromIndex(String jobId) { SearchResponse searchResponse = client().prepareSearch() + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) .setQuery(QueryBuilders.idsQuery().addIds(DataCounts.documentId(jobId))) .get(); if (searchResponse.getHits().getTotalHits().value != 1) { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml index eb3a73424a601..138612b66ab07 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml @@ -110,7 +110,8 @@ setup: - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: headers: @@ -462,7 +463,8 @@ setup: - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: headers: @@ -511,7 +513,8 @@ setup: - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: ml.delete_job: @@ -603,7 +606,8 @@ setup: - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: ml.delete_job: @@ -680,7 +684,8 @@ setup: - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: headers: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index ac2dd9557e5b2..51a3e2e8c050d 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -470,7 +470,8 @@ model_bytes: 10000000 - do: - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: catch: /Invalid update value for analysis_limits[:] model_memory_limit cannot be decreased below current usage; current usage \[9mb\], update had \[5mb\]/ @@ -926,7 +927,8 @@ key: value - do: - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: catch: /status_exception/ @@ -953,7 +955,8 @@ key: value - do: - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: catch: /status_exception/ @@ -990,7 +993,8 @@ } - do: - indices.refresh: {} + indices.refresh: + index: ["*",".ml-*"] - do: catch: /status_exception/