[ML] Make ml internal indices hidden

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -256,12 +256,13 @@ public Iterator<Setting<?>> settings() {
                       Setting.Property.Dynamic,
                       Setting.Property.IndexScope);
 
+    public static final String SETTING_INDEX_HIDDEN = "index.hidden";
     /**
      * Whether the index is considered hidden or not. A hidden index will not be resolved in
      * normal wildcard searches unless explicitly allowed
      */
     public static final Setting<Boolean> INDEX_HIDDEN_SETTING =
-        Setting.boolSetting("index.hidden", false, Property.IndexScope, Property.Final);
+        Setting.boolSetting(SETTING_INDEX_HIDDEN, false, Property.IndexScope, Property.Final);
 
     /**
      * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -119,9 +119,6 @@ public class MetaDataCreateIndexService {
      */
     private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(
         ".watch-history-*",
-        ".ml-anomalies-*",
-        ".ml-notifications-*",
-        ".ml-annotations*",
         ".data-frame-notifications-*",
         ".transform-notifications-*"
     ));

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java

@@ -618,9 +618,6 @@ public void testIndexNameExclusionsList() {
         // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty
         List<String> excludedNames = Arrays.asList(
             ".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
-            ".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
-            ".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
-            ".ml-annotations-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
             ".data-frame-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
             ".transform-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT)
         );

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java

@@ -59,11 +59,13 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
             // Create the annotations index if it doesn't exist already.
             if (mlLookup.containsKey(INDEX_NAME) == false) {
 
-                CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
-                createIndexRequest.mapping(annotationsMapping());
-                createIndexRequest.settings(Settings.builder()
-                    .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
-                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1"));
+                CreateIndexRequest createIndexRequest =
+                    new CreateIndexRequest(INDEX_NAME)
+                        .mapping(annotationsMapping())
+                        .settings(Settings.builder()
+                            .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+                            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
+                            .put(IndexMetaData.SETTING_INDEX_HIDDEN, true));
 
                 executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
                     ActionListener.<CreateIndexResponse>wrap(

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json

@@ -12,7 +12,8 @@
       "auto_expand_replicas" : "0-1",
       "query" : {
         "default_field" : "all_field_values"
-      }
+      },
+      "hidden": true
     }
   },
   "mappings": ${xpack.ml.anomalydetection.results.mappings}

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json

@@ -6,7 +6,8 @@
   ],
   "settings" : {
     "index" : {
-      "auto_expand_replicas" : "0-1"
+      "auto_expand_replicas" : "0-1",
+      "hidden": true
     }
   },
   "mappings" : {

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json

@@ -7,7 +7,8 @@
   "settings" : {
     "index" : {
       "number_of_shards" : "1",
-      "auto_expand_replicas" : "0-1"
+      "auto_expand_replicas" : "0-1",
+      "hidden": true
     }
   },
   "mappings" : {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -938,7 +938,6 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return List.of(
             new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, "Contains scheduling and anomaly tracking metadata"),
-            new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, "Contains ML model state"),
             new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, "Contains ML configuration data"),
             new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, "Contains ML model configuration and statistics")
         );

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
@@ -73,7 +74,10 @@ private boolean annotationsIndexExists() {
     private int numberOfAnnotationsAliases() {
         int count = 0;
         ImmutableOpenMap<String, List<AliasMetaData>> aliases = client().admin().indices()
-            .prepareGetAliases(AnnotationIndex.READ_ALIAS_NAME, AnnotationIndex.WRITE_ALIAS_NAME).get().getAliases();
+            .prepareGetAliases(AnnotationIndex.READ_ALIAS_NAME, AnnotationIndex.WRITE_ALIAS_NAME)
+            .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN)
+            .get()
+            .getAliases();
         if (aliases != null) {
             for (ObjectObjectCursor<String, List<AliasMetaData>> entry : aliases) {
                 count += entry.value.size();

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java

@@ -7,6 +7,7 @@
 
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -489,6 +490,7 @@ private void run(String jobId, CheckedRunnable<Exception> disrupt) throws Except
     // are what we expect them to be:
     private static DataCounts getDataCountsFromIndex(String jobId) {
         SearchResponse searchResponse = client().prepareSearch()
+                .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN)
                 .setQuery(QueryBuilders.idsQuery().addIds(DataCounts.documentId(jobId)))
                 .get();
         if (searchResponse.getHits().getTotalHits().value != 1) {

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml

@@ -470,7 +470,8 @@
           model_bytes: 10000000
 
   - do:
-      indices.refresh: {}
+      indices.refresh:
+        index: .ml-anomalies-shared
 
   - do:
       catch: /Invalid update value for analysis_limits[:] model_memory_limit cannot be decreased below current usage; current usage \[9mb\], update had \[5mb\]/
@@ -926,7 +927,8 @@
           key: value
 
   - do:
-      indices.refresh: {}
+      indices.refresh:
+        index: .ml-state
 
   - do:
       catch: /status_exception/
@@ -953,7 +955,8 @@
           key: value
 
   - do:
-      indices.refresh: {}
+      indices.refresh:
+        index: .ml-state
 
   - do:
       catch: /status_exception/
@@ -990,7 +993,8 @@
           }
 
   - do:
-      indices.refresh: {}
+      indices.refresh:
+        index: .ml-anomalies-shared
 
   - do:
       catch: /status_exception/