Cleaner MTurk cleanup (#315)

mephisto/abstractions/providers/mturk/mturk_utils.py

@@ -8,6 +8,7 @@
 import os
 import json
 import re
+from tqdm import tqdm
 from typing import Dict, Optional, Tuple, List, Any, TYPE_CHECKING
 from datetime import datetime
 
@@ -685,20 +686,23 @@ def get_outstanding_hits(client: MTurkClient) -> Dict[str, List[Dict[str, Any]]]
 
 
 def expire_and_dispose_hits(
-    client: MTurkClient, hits: List[Dict[str, Any]]
+    client: MTurkClient,
+    hits: List[Dict[str, Any]],
+    quiet: bool = False,
 ) -> List[Dict[str, Any]]:
     """
     Loops over attempting to expire and dispose any hits in the hits list that can be disposed
 
     Returns any HITs that could not be disposed of
     """
     non_disposed_hits = []
-    for h in hits:
+    for h in tqdm(hits, disable=quiet):
         try:
             client.delete_hit(HITId=h["HITId"])
-        except:
+        except Exception as e:
             client.update_expiration_for_hit(
                 HITId=h["HITId"], ExpireAt=datetime(2015, 1, 1)
             )
+            h["dispose_exception"] = e
             non_disposed_hits.append(h)
     return non_disposed_hits

mephisto/scripts/mturk/cleanup.py

@@ -86,4 +86,6 @@
         f"After disposing, {len(remaining_hits)} could not be disposed.\n"
         f"These may not have been reviewed yet, or are being actively worked on.\n"
         "They have been expired though, so please try to dispose later."
+        "The first 20 dispose errors are added below:"
     )
+    print([h["dispose_exception"] for h in remaining_hits[:20]])