Wipe Snapshots Before Indices in RestTests (#39662)

* Wipe Snapshots Before Indices in RestTests

* If we have a snapshot ongoing from the previous test and enter this method, then deleting the indices fails, which in turn fails the whole wipe
   * Fixed by first deleting/aborting snapshots

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.WarningsHandler;
+import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.PathUtils;
@@ -72,6 +73,7 @@
 import java.security.cert.CertificateException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -468,6 +470,8 @@ private void wipeCluster() throws Exception {
             waitForPendingRollupTasks();
         }
 
+        final Map<String, List<Map<?,?>>> inProgressSnapshots = wipeSnapshots();
+
         if (preserveIndicesUponCompletion() == false) {
             // wipe indices
             try {
@@ -508,8 +512,6 @@ private void wipeCluster() throws Exception {
             }
         }
 
-        wipeSnapshots();
-
         // wipe cluster settings
         if (preserveClusterSettings() == false) {
             wipeClusterSettings();
@@ -518,14 +520,18 @@ private void wipeCluster() throws Exception {
         if (hasXPack && false == preserveILMPoliciesUponCompletion()) {
             deleteAllPolicies();
         }
+
+        assertTrue("Found in progress snapshots [" + inProgressSnapshots + "].", inProgressSnapshots.isEmpty());
     }
 
     /**
      * Wipe fs snapshots we created one by one and all repositories so that the next test can create the repositories fresh and they'll
      * start empty. There isn't an API to delete all snapshots. There is an API to delete all snapshot repositories but that leaves all of
      * the snapshots intact in the repository.
+     * @return Map of repository name to list of snapshots found in unfinished state
      */
-    private void wipeSnapshots() throws IOException {
+    private Map<String, List<Map<?, ?>>> wipeSnapshots() throws IOException {
+        final Map<String, List<Map<?, ?>>> inProgressSnapshots = new HashMap<>();
         for (Map.Entry<String, ?> repo : entityAsMap(adminClient.performRequest(new Request("GET", "/_snapshot/_all"))).entrySet()) {
             String repoName = repo.getKey();
             Map<?, ?> repoSpec = (Map<?, ?>) repo.getValue();
@@ -538,6 +544,9 @@ private void wipeSnapshots() throws IOException {
                 for (Object snapshot : snapshots) {
                     Map<?, ?> snapshotInfo = (Map<?, ?>) snapshot;
                     String name = (String) snapshotInfo.get("snapshot");
+                    if (SnapshotsInProgress.State.valueOf((String) snapshotInfo.get("state")).completed() == false) {
+                        inProgressSnapshots.computeIfAbsent(repoName, key -> new ArrayList<>()).add(snapshotInfo);
+                    }
                     logger.debug("wiping snapshot [{}/{}]", repoName, name);
                     adminClient().performRequest(new Request("DELETE", "/_snapshot/" + repoName + "/" + name));
                 }
@@ -547,6 +556,7 @@ private void wipeSnapshots() throws IOException {
                 adminClient().performRequest(new Request("DELETE", "_snapshot/" + repoName));
             }
         }
+        return inProgressSnapshots;
     }
 
     /**