From e86e75f341612a25d6b37dc84277a301bb915eee Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 14 Nov 2023 11:53:08 -0500 Subject: [PATCH] Skipping bad json parse (#1990) Authors: - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Divye Gala (https://github.com/divyegala) URL: https://github.com/rapidsai/raft/pull/1990 --- .../src/raft-ann-bench/data_export/__main__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/raft-ann-bench/src/raft-ann-bench/data_export/__main__.py b/python/raft-ann-bench/src/raft-ann-bench/data_export/__main__.py index 3a547938ce..4978c99d60 100644 --- a/python/raft-ann-bench/src/raft-ann-bench/data_export/__main__.py +++ b/python/raft-ann-bench/src/raft-ann-bench/data_export/__main__.py @@ -52,9 +52,15 @@ def read_file(dataset, dataset_path, method): for file in os.listdir(dir): if file.endswith(".json"): with open(os.path.join(dir, file), "r") as f: - data = json.load(f) - df = pd.DataFrame(data["benchmarks"]) - yield (os.path.join(dir, file), file.split("-")[0], df) + try: + data = json.load(f) + df = pd.DataFrame(data["benchmarks"]) + yield os.path.join(dir, file), file.split("-")[0], df + except Exception as e: + print( + "An error occurred processing file %s (%s). " + "Skipping..." % (file, e) + ) def convert_json_to_csv_build(dataset, dataset_path):