diff --git a/jiant/proj/simple/runscript.py b/jiant/proj/simple/runscript.py
index b8bdae151..e7bace5fd 100644
--- a/jiant/proj/simple/runscript.py
+++ b/jiant/proj/simple/runscript.py
@@ -13,6 +13,7 @@
 import jiant.utils.zconf as zconf
 import jiant.utils.python.io as py_io
 from jiant.utils.python.logic import replace_none
+from jiant.utils.python.io import read_json
 
 
 @zconf.run_config
@@ -144,7 +145,11 @@ def run_simple(args: RunConfiguration, with_continue: bool = False):
                 if task_name in phase_task_list and not os.path.exists(
                     os.path.join(args.exp_dir, "cache", hf_config.model_type, task_name, phase)
                 ):
-                    phases_to_do.append(phase)
+                    config = read_json(task_config_path_dict[task_name])
+                    if phase in config["paths"]:
+                        phases_to_do.append(phase)
+                    else:
+                        phase_task_list.remove(task_name)
             if not phases_to_do:
                 continue
             print(f"Tokenizing Task '{task_name}' for phases '{','.join(phases_to_do)}'")