Make ML index aliases hidden

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java

@@ -44,9 +44,11 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
                                                          final ActionListener<Boolean> finalListener) {
 
         final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
-            final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
-                .addAlias(INDEX_NAME, READ_ALIAS_NAME)
-                .addAlias(INDEX_NAME, WRITE_ALIAS_NAME).request();
+            final IndicesAliasesRequest request =
+                client.admin().indices().prepareAliases()
+                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
+                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true))
+                    .request();
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
                 ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(r.isAcknowledged()), finalListener::onFailure),
                 client.admin().indices()::aliases);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java

@@ -136,7 +136,7 @@ private static void createFirstConcreteIndex(Client client,
             .indices()
             .prepareCreate(index);
         if (addAlias) {
-            requestBuilder.addAlias(new Alias(alias));
+            requestBuilder.addAlias(new Alias(alias).isHidden(true));
         }
         CreateIndexRequest request = requestBuilder.request();
 
@@ -166,7 +166,7 @@ private static void updateWriteAlias(Client client,
         IndicesAliasesRequestBuilder requestBuilder = client.admin()
             .indices()
             .prepareAliases()
-            .addAlias(newIndex, alias);
+            .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias).isHidden(true));
         if (currentIndex != null) {
             requestBuilder.removeAlias(currentIndex, alias);
         }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java

@@ -114,7 +114,7 @@ public void testCreateStateIndexAndAliasIfNecessary_CleanState() {
 
         CreateIndexRequest createRequest = createRequestCaptor.getValue();
         assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
-        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS))));
+        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true))));
     }
 
     private void assertNoClientInteractionsWhenWriteAliasAlreadyExists(String indexName) {
@@ -157,7 +157,7 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasAlreadyExistsAndPo
         assertThat(
             indicesAliasesRequest.getAliasActions(),
             contains(
-                AliasActions.add().alias(TEST_INDEX_ALIAS).index(FIRST_CONCRETE_INDEX),
+                AliasActions.add().alias(TEST_INDEX_ALIAS).index(FIRST_CONCRETE_INDEX).isHidden(true),
                 AliasActions.remove().alias(TEST_INDEX_ALIAS).index(LEGACY_INDEX_WITHOUT_SUFFIX)));
     }
 
@@ -175,7 +175,7 @@ private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List<String> e
         IndicesAliasesRequest indicesAliasesRequest = aliasesRequestCaptor.getValue();
         assertThat(
             indicesAliasesRequest.getAliasActions(),
-            contains(AliasActions.add().alias(TEST_INDEX_ALIAS).index(expectedWriteIndexName)));
+            contains(AliasActions.add().alias(TEST_INDEX_ALIAS).index(expectedWriteIndexName).isHidden(true)));
     }
 
     public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() {
@@ -205,7 +205,7 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLeg
 
         CreateIndexRequest createRequest = createRequestCaptor.getValue();
         assertThat(createRequest.index(), equalTo(FIRST_CONCRETE_INDEX));
-        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS))));
+        assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true))));
     }
 
     public void testIndexNameComparator() {

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -183,6 +184,7 @@ public void testDeleteExpiredData() throws Exception {
         retainAllSnapshots("snapshots-retention-with-retain");
 
         long totalModelSizeStatsBeforeDelete = client().prepareSearch("*")
+                .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
                 .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
                 .get().getHits().getTotalHits().value;
         long totalNotificationsCountBeforeDelete =
@@ -231,6 +233,7 @@ public void testDeleteExpiredData() throws Exception {
         assertThat(getModelSnapshots("results-and-snapshots-retention").size(), equalTo(1));
 
         long totalModelSizeStatsAfterDelete = client().prepareSearch("*")
+                .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
                 .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
                 .get().getHits().getTotalHits().value;
         long totalNotificationsCountAfterDelete =

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java

@@ -194,15 +194,17 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
         // appear immediately so wait here.
         assertBusy(() -> {
             try {
-                String aliasesResponse = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
-                assertThat(aliasesResponse,
-                        containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName) + "\":{\"aliases\":{"));
+                String aliasesResponse = getAliases();
+                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName)
+                        + "\":{\"aliases\":{"));
                 assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)
-                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}}}"));
-                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1) + "\":{}"));
+                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}},\"is_hidden\":true}"));
+                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1)
+                        + "\":{\"is_hidden\":true}"));
                 assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)
-                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}}}"));
-                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2) + "\":{}"));
+                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}},\"is_hidden\":true}"));
+                assertThat(aliasesResponse, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2)
+                        + "\":{\"is_hidden\":true}"));
             } catch (ResponseException e) {
                 throw new AssertionError(e);
             }
@@ -270,7 +272,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
         client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1));
 
         // check that indices still exist, but no longer have job1 entries and aliases are gone
-        responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
+        responseAsString = getAliases();
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
         assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists
 
@@ -286,7 +288,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
 
         // Delete the second job and verify aliases are gone, and original concrete/custom index is gone
         client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2));
-        responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_aliases")).getEntity());
+        responseAsString = getAliases();
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
 
         refreshAllIndices();
@@ -626,6 +628,7 @@ public void testMultiIndexDelete() throws Exception {
         extraIndex1.setJsonEntity("{\n" +
             "    \"aliases\" : {\n" +
             "        \"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)+ "\" : {\n" +
+            "            \"is_hidden\" : true,\n" +
             "            \"filter\" : {\n" +
             "                \"term\" : {\"" + Job.ID + "\" : \"" + jobId + "\" }\n" +
             "            }\n" +
@@ -637,6 +640,7 @@ public void testMultiIndexDelete() throws Exception {
         extraIndex2.setJsonEntity("{\n" +
             "    \"aliases\" : {\n" +
             "        \"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)+ "\" : {\n" +
+            "            \"is_hidden\" : true,\n" +
             "            \"filter\" : {\n" +
             "                \"term\" : {\"" + Job.ID + "\" : \"" + jobId + "\" }\n" +
             "            }\n" +
@@ -784,6 +788,9 @@ public void testDelete_multipleRequest() throws Exception {
             assertNull(recreationException.get().getMessage(), recreationException.get());
         }
 
+        String expectedReadAliasString = "\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
+            + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}},\"is_hidden\":true}";
+        String expectedWriteAliasString = "\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{\"is_hidden\":true}";
         try {
             // The idea of the code above is that the deletion is sufficiently time-consuming that
             // all threads enter the deletion call before the first one exits it.  Usually this happens,
@@ -796,9 +803,8 @@ public void testDelete_multipleRequest() throws Exception {
             // if there's been a race between deletion and recreation these are what will be missing.
             String aliases = getAliases();
 
-            assertThat(aliases, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
-                    + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}}}"));
-            assertThat(aliases, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{}"));
+            assertThat(aliases, containsString(expectedReadAliasString));
+            assertThat(aliases, containsString(expectedWriteAliasString));
 
 
         } catch (ResponseException missingJobException) {
@@ -807,9 +813,8 @@ public void testDelete_multipleRequest() throws Exception {
 
             // The job aliases should be deleted
             String aliases = getAliases();
-            assertThat(aliases, not(containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId)
-                    + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId + "\",\"boost\":1.0}}}}")));
-            assertThat(aliases, not(containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId) + "\":{}")));
+            assertThat(aliases, not(containsString(expectedReadAliasString)));
+            assertThat(aliases, not(containsString(expectedWriteAliasString)));
         }
 
         assertEquals(numThreads, recreationGuard.get());

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java

@@ -489,7 +489,7 @@ private void deleteAliases(ParentTaskAssigningClient parentTaskClient, String jo
 
         // first find the concrete indices associated with the aliases
         GetAliasesRequest aliasesRequest = new GetAliasesRequest().aliases(readAliasName, writeAliasName)
-                .indicesOptions(IndicesOptions.lenientExpandOpen());
+                .indicesOptions(IndicesOptions.lenientExpandOpenHidden());
         executeAsyncWithOrigin(parentTaskClient.threadPool().getThreadContext(), ML_ORIGIN, aliasesRequest,
                 ActionListener.<GetAliasesResponse>wrap(
                         getAliasesResponse -> {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java

@@ -282,9 +282,16 @@ public void createJobResultIndex(Job job, ClusterState state, final ActionListen
         final String indexName = tempIndexName;
 
         ActionListener<Boolean> indexAndMappingsListener = ActionListener.wrap(success -> {
-            final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
-                    .addAlias(indexName, readAliasName, QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId()))
-                    .addAlias(indexName, writeAliasName).request();
+            final IndicesAliasesRequest request =
+                client.admin().indices().prepareAliases()
+                    .addAliasAction(
+                        IndicesAliasesRequest.AliasActions.add()
+                            .index(indexName)
+                            .alias(readAliasName)
+                            .isHidden(true)
+                            .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId())))
+                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(writeAliasName).isHidden(true))
+                    .request();
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
                     ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(true), finalListener::onFailure),
                     client.admin().indices()::aliases);

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java

@@ -19,6 +19,8 @@
 
 import java.util.List;
 
+import static org.hamcrest.Matchers.is;
+
 public class AnnotationIndexIT extends MlSingleNodeTestCase {
 
     @Override
@@ -72,6 +74,9 @@ private int numberOfAnnotationsAliases() {
             .getAliases();
         if (aliases != null) {
             for (ObjectObjectCursor<String, List<AliasMetaData>> entry : aliases) {
+                for (AliasMetaData aliasMetaData : entry.value) {
+                    assertThat("Annotations aliases should be hidden but are not: " + aliases, aliasMetaData.isHidden(), is(true));
+                }
                 count += entry.value.size();
             }
         }

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java

@@ -142,8 +142,10 @@ public void testPutJob_CreatesResultsIndex() {
         assertThat(mappingProperties.keySet(), hasItem("by_field_1"));
 
         // Check aliases have been created
-        assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()),
-            AnomalyDetectorsIndex.resultsWriteAlias(job1.getId())));
+        assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(
+            AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()),
+            AnomalyDetectorsIndex.resultsWriteAlias(job1.getId())
+        ));
 
         // Now let's create a second job to test things work when the index exists already
         assertThat(mappingProperties.keySet(), not(hasItem("by_field_2")));
@@ -187,8 +189,10 @@ public void testPutJob_WithCustomResultsIndex() {
         assertThat(mappingProperties.keySet(), hasItem("by_field"));
 
         // Check aliases have been created
-        assertThat(getAliases(customIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()),
-            AnomalyDetectorsIndex.resultsWriteAlias(job.getId())));
+        assertThat(getAliases(customIndex), containsInAnyOrder(
+            AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()),
+            AnomalyDetectorsIndex.resultsWriteAlias(job.getId())
+        ));
     }
 
     @AwaitsFix(bugUrl ="https://github.com/elastic/elasticsearch/issues/40134")
@@ -366,12 +370,14 @@ private Map<String, Object> getIndexMappingProperties(String index) {
     }
 
     private Set<String> getAliases(String index) {
-        GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(
-            new GetAliasesRequest().indices(index)).actionGet();
+        GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(new GetAliasesRequest().indices(index)).actionGet();
         ImmutableOpenMap<String, List<AliasMetaData>> aliases = getAliasesResponse.getAliases();
         assertThat(aliases.containsKey(index), is(true));
-        List<AliasMetaData> aliasMetaData = aliases.get(index);
-        return aliasMetaData.stream().map(AliasMetaData::alias).collect(Collectors.toSet());
+        List<AliasMetaData> aliasMetaDataList = aliases.get(index);
+        for (AliasMetaData aliasMetaData : aliasMetaDataList) {
+            assertThat("Anomalies aliases should be hidden but are not: " + aliases, aliasMetaData.isHidden(), is(true));
+        }
+        return aliasMetaDataList.stream().map(AliasMetaData::alias).collect(Collectors.toSet());
     }
 
     private List<Calendar> getCalendars(String jobId) throws Exception {

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml

@@ -148,6 +148,7 @@ setup:
 
   - do:
       search:
+        expand_wildcards: all
         rest_total_hits_as_int: true
         body: { query: { bool: { must: [
               { query_string: { query: "result_type:record"} },

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml

@@ -100,6 +100,13 @@ setup:
         index: ".ml-state-000001"
   - is_true: ''
 
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      indices.exists_alias:
+        name: ".ml-state-write"
+  - is_true: ''
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -439,6 +446,13 @@ setup:
         index: ".ml-state-000001"
   - is_true: ''
 
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      indices.exists_alias:
+        name: ".ml-state-write"
+  - is_true: ''
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -681,6 +695,13 @@ setup:
         index: ".ml-state-000001"
   - is_true: ''
 
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      indices.exists_alias:
+        name: ".ml-state-write"
+  - is_true: ''
+
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml

@@ -70,7 +70,7 @@
   - do:
       indices.get_alias:
         name: ".ml-anomalies-.write-job-crud-test-apis"
-  - match: { \.ml-anomalies-shared.aliases.\.ml-anomalies-\.write-job-crud-test-apis: {} }
+  - match: { \.ml-anomalies-shared.aliases.\.ml-anomalies-\.write-job-crud-test-apis: { is_hidden: true } }
 
   - do:
       ml.delete_job: