Skip to content

Commit

Permalink
[TEST] Work around _cat/indices bug with security enabled (#47160)
Browse files Browse the repository at this point in the history 
When the ML native multi-node tests use _cat/indices/_all
and the request goes to a non-master node, _all is
translated to a list of concrete indices by the authz layer
on the coordinating node before the request is forwarded
to the master node. Then it is possible for the master
node to return an index_not_found_exception if one of
the concrete indices that was expanded on the
coordinating node has been deleted in the meantime.
(#47159 has been opened to track the underlying problem.)

It has been observed that the index that gets deleted when
the problem affects the ML native multi-node tests is
always the ML notifications index. The tests that fail are
only interested in the presence or absense of ML results
indices. Therefore the workaround is to only _cat indices
that match the ML results index pattern.

Fixes #45652
  • Loading branch information
@droberts195
droberts195 committed Sep 26, 2019
1 parent ddadcd8 commit cca2343
Showing 1 changed file with 27 additions and 13 deletions.
40 changes: 27 additions & 13 deletions ...native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,9 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
}
});

String responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
// Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(responseAsString,
containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
Expand Down Expand Up @@ -270,7 +272,8 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists

responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));

client().performRequest(new Request("POST", "/_refresh"));
Expand All @@ -285,7 +288,8 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));

client().performRequest(new Request("POST", "/_refresh"));
responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
responseAsString = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)));
}

Expand Down Expand Up @@ -392,19 +396,21 @@ public void testCreateJob_WithClashingFieldMappingsFails() throws Exception {
"avoid the clash by assigning a dedicated results index"));
}

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/45652")
public void testDeleteJob() throws Exception {
String jobId = "delete-job-job";
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId);

String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
// Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesBeforeDelete, containsString(indexName));

client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));

// check that the index still exists (it's shared by default)
String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
String indicesAfterDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesAfterDelete, containsString(indexName));

waitUntilIndexIsEmpty(indexName);
Expand All @@ -414,13 +420,14 @@ public void testDeleteJob() throws Exception {
client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")));
}

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/45652")
public void testDeleteJobAsync() throws Exception {
String jobId = "delete-job-async-job";
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId);

String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
// Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesBeforeDelete, containsString(indexName));

Response response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId
Expand All @@ -432,7 +439,8 @@ public void testDeleteJobAsync() throws Exception {
assertThat(EntityUtils.toString(taskResponse.getEntity()), containsString("\"acknowledged\":true"));

// check that the index still exists (it's shared by default)
String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
String indicesAfterDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesAfterDelete, containsString(indexName));

waitUntilIndexIsEmpty(indexName);
Expand Down Expand Up @@ -467,7 +475,9 @@ public void testDeleteJobAfterMissingIndex() throws Exception {
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
createFarequoteJob(jobId);

String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
// Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesBeforeDelete, containsString(indexName));

// Manually delete the index so that we can test that deletion proceeds
Expand All @@ -477,7 +487,8 @@ public void testDeleteJobAfterMissingIndex() throws Exception {
client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));

// check index was deleted
String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
String indicesAfterDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesAfterDelete, not(containsString(aliasName)));
assertThat(indicesAfterDelete, not(containsString(indexName)));

Expand Down Expand Up @@ -547,7 +558,9 @@ public void testMultiIndexDelete() throws Exception {
"}");
client().performRequest(extraIndex2);

String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
// Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652
String indicesBeforeDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesBeforeDelete, containsString(indexName));
assertThat(indicesBeforeDelete, containsString(indexName + "-001"));
assertThat(indicesBeforeDelete, containsString(indexName + "-002"));
Expand Down Expand Up @@ -586,7 +599,8 @@ public void testMultiIndexDelete() throws Exception {
client().performRequest(new Request("POST", "/_refresh"));

// check that the indices still exist but are empty
String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity());
String indicesAfterDelete = EntityUtils.toString(client().performRequest(
new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity());
assertThat(indicesAfterDelete, containsString(indexName));
assertThat(indicesAfterDelete, containsString(indexName + "-001"));
assertThat(indicesAfterDelete, containsString(indexName + "-002"));
Expand Down

0 comments on commit cca2343

Please sign in to comment.