diff --git a/mephisto/abstractions/providers/mturk/mturk_utils.py b/mephisto/abstractions/providers/mturk/mturk_utils.py index 8576af911..a1eb4f93a 100644 --- a/mephisto/abstractions/providers/mturk/mturk_utils.py +++ b/mephisto/abstractions/providers/mturk/mturk_utils.py @@ -8,6 +8,7 @@ import os import json import re +from tqdm import tqdm from typing import Dict, Optional, Tuple, List, Any, TYPE_CHECKING from datetime import datetime @@ -685,7 +686,9 @@ def get_outstanding_hits(client: MTurkClient) -> Dict[str, List[Dict[str, Any]]] def expire_and_dispose_hits( - client: MTurkClient, hits: List[Dict[str, Any]] + client: MTurkClient, + hits: List[Dict[str, Any]], + quiet: bool = False, ) -> List[Dict[str, Any]]: """ Loops over attempting to expire and dispose any hits in the hits list that can be disposed @@ -693,12 +696,13 @@ def expire_and_dispose_hits( Returns any HITs that could not be disposed of """ non_disposed_hits = [] - for h in hits: + for h in tqdm(hits, disable=quiet): try: client.delete_hit(HITId=h["HITId"]) - except: + except Exception as e: client.update_expiration_for_hit( HITId=h["HITId"], ExpireAt=datetime(2015, 1, 1) ) + h["dispose_exception"] = e non_disposed_hits.append(h) return non_disposed_hits diff --git a/mephisto/scripts/mturk/cleanup.py b/mephisto/scripts/mturk/cleanup.py index ceb6504af..4053e74ff 100644 --- a/mephisto/scripts/mturk/cleanup.py +++ b/mephisto/scripts/mturk/cleanup.py @@ -86,4 +86,6 @@ f"After disposing, {len(remaining_hits)} could not be disposed.\n" f"These may not have been reviewed yet, or are being actively worked on.\n" "They have been expired though, so please try to dispose later." + "The first 20 dispose errors are added below:" ) + print([h["dispose_exception"] for h in remaining_hits[:20]])