some cleanup

evals/multimetric_qa_eval_run.py

@@ -1,26 +1,45 @@
 import subprocess
 import json
 import argparse
+import os
+from typing import List
+import sys
 
 
-def main():
-    parser = argparse.ArgumentParser()
+def run_command(command: List[str]):
+    try:
+        process = subprocess.Popen(
+            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
+        )
 
-    parser.add_argument(
-        "--params_file", type=str, required=True, help="Which dataset to evaluate on"
-    )
-    parser.add_argument("--out_dir", type=str, help="Dir to save eval results")
+        while True:
+            stdout_line = process.stdout.readline()
+            stderr_line = process.stderr.readline()
 
-    args = parser.parse_args()
+            if stdout_line == "" and stderr_line == "" and process.poll() is not None:
+                break
+
+            if stdout_line:
+                print(stdout_line.rstrip())
+            if stderr_line:
+                print(f"Error: {stderr_line.rstrip()}", file=sys.stderr)
 
-    with open(args.params_file, "r") as file:
+        if process.returncode != 0:
+            raise subprocess.CalledProcessError(process.returncode, command)
+    finally:
+        process.stdout.close()
+        process.stderr.close()
+
+
+def run_evals_for_paramsfile(params_file, out_dir):
+    with open(params_file, "r") as file:
         parameters = json.load(file)
 
     for metric in parameters["metric_names"]:
         params = parameters
         params["metric_names"] = [metric]
 
-        temp_paramfile = args.params_file.replace(".json", f"_{metric}.json")
+        temp_paramfile = params_file.replace(".json", f"_{metric}.json")
         with open(temp_paramfile, "w") as file:
             json.dump(params, file)
 
@@ -30,17 +49,26 @@ def main():
             "--params_file",
             temp_paramfile,
             "--out_dir",
-            args.out_dir,
+            out_dir,
         ]
 
-        process = subprocess.Popen(
-            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
-        )
+        run_command(command)
+
+        if os.path.exists(temp_paramfile):
+            os.remove(temp_paramfile)
+
+
+def main():
+    parser = argparse.ArgumentParser()
 
-        for line in process.stdout:
-            print(line, end="")  # Print output line-by-line
+    parser.add_argument(
+        "--params_file", type=str, required=True, help="Which dataset to evaluate on"
+    )
+    parser.add_argument("--out_dir", type=str, help="Dir to save eval results")
+
+    args = parser.parse_args()
 
-        process.wait()
+    run_evals_for_paramsfile(args.params_file, args.out_dir)
 
 
 if __name__ == "__main__":

evals/qa_eval_utils.py

@@ -45,6 +45,7 @@ def save_results_as_image(results, out_path):
             for rag_option, metric_data in table_data.items():
                 for name, value in metric_data.items():
                     metric_name = name
+                    break
             df = pd.DataFrame.from_dict(table_data, orient="index")
             df.index.name = f"Dataset: {dataset}, Num Samples: {num_samples}"
             image_path = out_path / Path(f"table_{dataset}_{num_samples}_{metric_name}.png")