From 02ab99dbe4123ac4377f07bc795567e8c40a7371 Mon Sep 17 00:00:00 2001 From: Nitin Date: Thu, 16 Mar 2023 19:44:33 +0530 Subject: [PATCH 1/8] Additional requirements listed --- requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3929705..9e36de5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,8 @@ surgeon_pytorch lmdb pyarrow pandas -pillow \ No newline at end of file +pillow +opencv-python +scikit-learn +scikit-image +Wand \ No newline at end of file From 7d23ead9aa7772771737b69e80c42b282bad8719 Mon Sep 17 00:00:00 2001 From: Nitin Date: Thu, 16 Mar 2023 19:45:28 +0530 Subject: [PATCH 2/8] Fixed data links for Worst case task --- shifthappens/tasks/worst_case/worst_case.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shifthappens/tasks/worst_case/worst_case.py b/shifthappens/tasks/worst_case/worst_case.py index b5872e8..a4ea9a7 100644 --- a/shifthappens/tasks/worst_case/worst_case.py +++ b/shifthappens/tasks/worst_case/worst_case.py @@ -35,13 +35,13 @@ class WorstCase(Task): [ "worstcase", "restricted_superclass.csv", - "https://anonymous.4open.science/r/worst_classes-B94C/restricted_superclass.csv", + "https://raw.githubusercontent.com/j-cb/worst_classes/master/restricted_superclass.csv", None, ], [ "worstcase", "new_labels.csv", - "https://anonymous.4open.science/r/worst_classes-B94C/new_labels.csv", + "https://raw.githubusercontent.com/j-cb/worst_classes/master/new_labels.csv", None, ], ) From b1dfc5a54319fe0ec840de4a3d0ff1297b4be8d1 Mon Sep 17 00:00:00 2001 From: Nitin Date: Sat, 15 Apr 2023 20:42:17 +0200 Subject: [PATCH 3/8] Added functions to convert results to json and vice versa --- shifthappens/task_data/task_metadata.py | 18 +++++++++++++++ shifthappens/tasks/task_result.py | 29 +++++++++++++++++++++++++ shifthappens/utils.py | 16 +++++++++++++- 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/shifthappens/task_data/task_metadata.py b/shifthappens/task_data/task_metadata.py index 9c69067..d5148cc 100644 --- a/shifthappens/task_data/task_metadata.py +++ b/shifthappens/task_data/task_metadata.py @@ -1,6 +1,7 @@ """Class for storing a task's metadata.""" from dataclasses import dataclass +import json @dataclass(frozen=True, eq=True) @@ -22,5 +23,22 @@ class TaskMetadata: relative_data_folder: str standalone: bool = True + def serialize_task_metadata(self) -> str: + metadata_dict = { + 'name': self.name, + 'relative_data_folder': self.relative_data_folder, + 'standalone': self.standalone + } + return json.dumps(metadata_dict) + + @staticmethod + def deserialize_task_metadata(metadata_str: str): + metadata_dict = json.loads(metadata_str) + metadata = TaskMetadata( + name=metadata_dict['name'], + relative_data_folder= metadata_dict['relative_data_folder'], + standalone=metadata_dict['standalone'] + ) + return metadata _TASK_METADATA_FIELD = "__task_metadata__" diff --git a/shifthappens/tasks/task_result.py b/shifthappens/tasks/task_result.py index 4208826..ed44d5b 100644 --- a/shifthappens/tasks/task_result.py +++ b/shifthappens/tasks/task_result.py @@ -64,3 +64,32 @@ def __getattr__(self, item) -> float: return self[item] else: return super().__getattribute__(item) + + def serialize_summary_metrics(self) -> str: + return str({key.name:value for (key, value) in self.summary_metrics.items()}) + + + def serialize_task_result(self) -> str: + result_dict = { + 'summary_metrics': self.serialize_summary_metrics(), + 'metrics': str(self._metrics) + } + return str(result_dict) + + @staticmethod + def deserialize_summary_metrics(summary_metrics_str:str) -> Dict[Metric, Union[str, Tuple[str, ...]]]: + summary_metrics = eval(summary_metrics_str) + result = {} + for (key,value) in summary_metrics.items(): + result[Metric.__members__.get(key)] = value + return result + + @staticmethod + def deserialize_task_result(task_result_str: str): + result_dict = eval(task_result_str) + metrics = eval(result_dict['metrics']) + summary_metrics = TaskResult.deserialize_summary_metrics(result_dict['summary_metrics']) + return TaskResult( + summary_metrics=summary_metrics, + **metrics + ) diff --git a/shifthappens/utils.py b/shifthappens/utils.py index 57d696d..6d0771d 100644 --- a/shifthappens/utils.py +++ b/shifthappens/utils.py @@ -1,12 +1,16 @@ """Utility functions that are needed for the entire package.""" import errno +import json import os import sys import time import urllib.error from itertools import product -from typing import Optional +from typing import Dict, Optional, Union + +from shifthappens.task_data import task_metadata +from shifthappens.tasks.task_result import TaskResult def dict_product(d): @@ -135,3 +139,13 @@ def download_and_extract_archive( archive = os.path.join(data_folder, filename) print(f"Extracting {archive} to {data_folder}") tv_utils.extract_archive(archive, data_folder, remove_finished) + +def serialize_model_results(results: Dict[task_metadata.TaskMetadata, Union[TaskResult, None]]) -> str: + return json.dumps({key.serialize_task_metadata():value.serialize_task_result() for (key,value) in results.items() if value != None}) + +def deserialize_model_results(results_str) -> Dict[task_metadata.TaskMetadata, TaskResult]: + results_json_dict = json.loads(results_str) + results = {} + for (key,value) in results_json_dict.items(): + results[task_metadata.TaskMetadata.deserialize_task_metadata(key)] = TaskResult.deserialize_task_result(value) + return results From 49b0f7b96c422ace62b4d26b90cd3eda05a775fe Mon Sep 17 00:00:00 2001 From: Nitin Date: Sat, 15 Apr 2023 20:51:52 +0200 Subject: [PATCH 4/8] Added docstrings to methods --- shifthappens/task_data/task_metadata.py | 6 ++++++ shifthappens/tasks/task_result.py | 12 ++++++++++++ shifthappens/utils.py | 6 ++++++ 3 files changed, 24 insertions(+) diff --git a/shifthappens/task_data/task_metadata.py b/shifthappens/task_data/task_metadata.py index d5148cc..fdbf207 100644 --- a/shifthappens/task_data/task_metadata.py +++ b/shifthappens/task_data/task_metadata.py @@ -24,6 +24,9 @@ class TaskMetadata: standalone: bool = True def serialize_task_metadata(self) -> str: + """ + Serialize TaskMetadata object into json string. + """ metadata_dict = { 'name': self.name, 'relative_data_folder': self.relative_data_folder, @@ -33,6 +36,9 @@ def serialize_task_metadata(self) -> str: @staticmethod def deserialize_task_metadata(metadata_str: str): + """ + Deserialize valid json string into TaskMetadata object. + """ metadata_dict = json.loads(metadata_str) metadata = TaskMetadata( name=metadata_dict['name'], diff --git a/shifthappens/tasks/task_result.py b/shifthappens/tasks/task_result.py index ed44d5b..5eb8095 100644 --- a/shifthappens/tasks/task_result.py +++ b/shifthappens/tasks/task_result.py @@ -66,10 +66,16 @@ def __getattr__(self, item) -> float: return super().__getattribute__(item) def serialize_summary_metrics(self) -> str: + """ + Serializes summary metrics of the objects into a string. + """ return str({key.name:value for (key, value) in self.summary_metrics.items()}) def serialize_task_result(self) -> str: + """ + Serializes TaskResult object into a string. + """ result_dict = { 'summary_metrics': self.serialize_summary_metrics(), 'metrics': str(self._metrics) @@ -78,6 +84,9 @@ def serialize_task_result(self) -> str: @staticmethod def deserialize_summary_metrics(summary_metrics_str:str) -> Dict[Metric, Union[str, Tuple[str, ...]]]: + """ + Deserializes valid string into summary_metrics. + """ summary_metrics = eval(summary_metrics_str) result = {} for (key,value) in summary_metrics.items(): @@ -86,6 +95,9 @@ def deserialize_summary_metrics(summary_metrics_str:str) -> Dict[Metric, Union[s @staticmethod def deserialize_task_result(task_result_str: str): + """ + Deserializes valid string into a TaskResult object. + """ result_dict = eval(task_result_str) metrics = eval(result_dict['metrics']) summary_metrics = TaskResult.deserialize_summary_metrics(result_dict['summary_metrics']) diff --git a/shifthappens/utils.py b/shifthappens/utils.py index 6d0771d..77a9542 100644 --- a/shifthappens/utils.py +++ b/shifthappens/utils.py @@ -141,9 +141,15 @@ def download_and_extract_archive( tv_utils.extract_archive(archive, data_folder, remove_finished) def serialize_model_results(results: Dict[task_metadata.TaskMetadata, Union[TaskResult, None]]) -> str: + """ + Converts evaluation results of a model into json objects. + """ return json.dumps({key.serialize_task_metadata():value.serialize_task_result() for (key,value) in results.items() if value != None}) def deserialize_model_results(results_str) -> Dict[task_metadata.TaskMetadata, TaskResult]: + """ + Converts json objects to a dictionary with (TaskMetadata, TaskResult) as (key, value) + """ results_json_dict = json.loads(results_str) results = {} for (key,value) in results_json_dict.items(): From 2cdcc4a5f8e74fbfff273eceb04f7004050e3dac Mon Sep 17 00:00:00 2001 From: Nitin Date: Sat, 29 Apr 2023 14:37:49 +0200 Subject: [PATCH 5/8] Fixing linting errors --- shifthappens/task_data/task_metadata.py | 2 +- shifthappens/tasks/task_result.py | 6 +++--- shifthappens/utils.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shifthappens/task_data/task_metadata.py b/shifthappens/task_data/task_metadata.py index fdbf207..aaf6eb7 100644 --- a/shifthappens/task_data/task_metadata.py +++ b/shifthappens/task_data/task_metadata.py @@ -42,7 +42,7 @@ def deserialize_task_metadata(metadata_str: str): metadata_dict = json.loads(metadata_str) metadata = TaskMetadata( name=metadata_dict['name'], - relative_data_folder= metadata_dict['relative_data_folder'], + relative_data_folder=metadata_dict['relative_data_folder'], standalone=metadata_dict['standalone'] ) return metadata diff --git a/shifthappens/tasks/task_result.py b/shifthappens/tasks/task_result.py index 5eb8095..cfa258a 100644 --- a/shifthappens/tasks/task_result.py +++ b/shifthappens/tasks/task_result.py @@ -69,7 +69,7 @@ def serialize_summary_metrics(self) -> str: """ Serializes summary metrics of the objects into a string. """ - return str({key.name:value for (key, value) in self.summary_metrics.items()}) + return str({key.name: value for (key, value) in self.summary_metrics.items()}) def serialize_task_result(self) -> str: @@ -83,13 +83,13 @@ def serialize_task_result(self) -> str: return str(result_dict) @staticmethod - def deserialize_summary_metrics(summary_metrics_str:str) -> Dict[Metric, Union[str, Tuple[str, ...]]]: + def deserialize_summary_metrics(summary_metrics_str: str) -> Dict[Metric, Union[str, Tuple[str, ...]]]: """ Deserializes valid string into summary_metrics. """ summary_metrics = eval(summary_metrics_str) result = {} - for (key,value) in summary_metrics.items(): + for (key, value) in summary_metrics.items(): result[Metric.__members__.get(key)] = value return result diff --git a/shifthappens/utils.py b/shifthappens/utils.py index 77a9542..993f9e4 100644 --- a/shifthappens/utils.py +++ b/shifthappens/utils.py @@ -144,7 +144,7 @@ def serialize_model_results(results: Dict[task_metadata.TaskMetadata, Union[Task """ Converts evaluation results of a model into json objects. """ - return json.dumps({key.serialize_task_metadata():value.serialize_task_result() for (key,value) in results.items() if value != None}) + return json.dumps({key.serialize_task_metadata(): value.serialize_task_result() for (key, value) in results.items() if value is not None}) def deserialize_model_results(results_str) -> Dict[task_metadata.TaskMetadata, TaskResult]: """ @@ -152,6 +152,6 @@ def deserialize_model_results(results_str) -> Dict[task_metadata.TaskMetadata, T """ results_json_dict = json.loads(results_str) results = {} - for (key,value) in results_json_dict.items(): + for (key, value) in results_json_dict.items(): results[task_metadata.TaskMetadata.deserialize_task_metadata(key)] = TaskResult.deserialize_task_result(value) return results From 65f82bfc24f5fe2a8d53c0c98f626c13741eef6b Mon Sep 17 00:00:00 2001 From: Nitin Date: Sat, 29 Apr 2023 14:51:06 +0200 Subject: [PATCH 6/8] Functionality to control the tasks and models to be run through a json config file --- make_script.py | 35 +++++++++++++++++++++++++++++++++++ run.sh | 2 ++ tasks_config.json | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 make_script.py create mode 100644 run.sh create mode 100644 tasks_config.json diff --git a/make_script.py b/make_script.py new file mode 100644 index 0000000..a37a25e --- /dev/null +++ b/make_script.py @@ -0,0 +1,35 @@ +import json + +with open('./tasks_config.json', 'r') as f: + tasks_config = json.load(f) + +scriptstring = """ +from datetime import datetime +import shifthappens.benchmark +import shifthappens.utils +""" + +print(tasks_config['tasks']) +for task in tasks_config['tasks']: + scriptstring += tasks_config['import_lines'][task] + scriptstring += "\n" + +scriptstring += tasks_config['import_lines'][tasks_config['model']] + +out_file_location = tasks_config['out_file_location'] +relative_data_folder = tasks_config['relative_data_folder'] + +scriptstring += f""" +tasks = shifthappens.benchmark.get_task_registrations() +model = {tasks_config['model']}() +results = shifthappens.benchmark.evaluate_model( + model, "{relative_data_folder}" +) +results_string = shifthappens.utils.serialize_model_results(results) +out_file_location = "{out_file_location}" +with open(out_file_location, 'w') as outfile: + outfile.write(results_string) +""" +with open('./run_tasks.py', 'w') as run_script_file: + run_script_file.write(scriptstring) + diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..95e7c74 --- /dev/null +++ b/run.sh @@ -0,0 +1,2 @@ +python3 make_script.py +python3 run_tasks.py \ No newline at end of file diff --git a/tasks_config.json b/tasks_config.json new file mode 100644 index 0000000..200ebb7 --- /dev/null +++ b/tasks_config.json @@ -0,0 +1,40 @@ +{ + "out_file_location": "/mnt/qb/work/bethge/<>/<>", + "relative_data_folder": "/mnt/qb/work/bethge/<>/<>", + "tasks": [ + "imagenet_c", + "ccc", + "imagenet_3dcc", + "imagenet_cartoon", + "imagenet_d", + "imagenet_drawing", + "imagenet_m", + "imagenet_metashift", + "imagenet_patch", + "imagenet_r", + "objectnet", + "raccoons_ood", + "siscore", + "ssb", + "worst_case" + ], + "model": "ResNet18", + "import_lines": { + "imagenet_c": "from shifthappens.tasks.imagenet_c.imagenet_c import ImageNetCSeparateCorruptions", + "ccc": "from shifthappens.tasks.ccc.ccc import CCC", + "imagenet_3dcc": "from shifthappens.tasks.imagenet_3dcc.imagenet_3dcc import ImageNet3DCCSeparateCorruptions", + "imagenet_cartoon": "from shifthappens.tasks.imagenet_cartoon.imagenet_cartoon import ImageNetCartoon", + "imagenet_d": "from shifthappens.tasks.imagenet_d.imagenet_d import *", + "imagenet_drawing": "from shifthappens.tasks.imagenet_drawing.imagenet_drawing import ImageNetDrawing", + "imagenet_m": "from shifthappens.tasks.imagenet_m.imagenet_m import ImageNetM", + "imagenet_metashift": "from shifthappens.tasks.imagenet_metashift.imagenet_metashift import ImageNetMetaShift", + "imagenet_patch": "from shifthappens.tasks.imagenet_patch.imagenet_patch import ImageNetPatchCorruptions", + "imagenet_r": "from shifthappens.tasks.imagenet_r.imagenet_r import ImageNetR", + "objectnet": "from shifthappens.tasks.objectnet.objectnet import ObjectNet", + "raccoons_ood": "from shifthappens.tasks.raccoons_ood.raccoons_ood import RaccOOD", + "siscore": "from shifthappens.tasks.siscore.siscore import *", + "ssb": "from shifthappens.tasks.ssb.semantic_shift_benchmark import *", + "worst_case": "from shifthappens.tasks.worst_case.worst_case import WorstCase", + "ResNet18": "from shifthappens.models.torchvision import ResNet18" + } +} \ No newline at end of file From b07b0ee6c06d91659f7b1453d458b975dcb97388 Mon Sep 17 00:00:00 2001 From: Nitin Date: Sat, 29 Apr 2023 15:33:57 +0200 Subject: [PATCH 7/8] Formatting changes --- make_script.py | 17 ++++++++--------- shifthappens/task_data/task_metadata.py | 13 +++++++------ shifthappens/tasks/task_result.py | 20 ++++++++++---------- shifthappens/utils.py | 25 +++++++++++++++++++------ 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/make_script.py b/make_script.py index a37a25e..43353b8 100644 --- a/make_script.py +++ b/make_script.py @@ -1,6 +1,6 @@ import json -with open('./tasks_config.json', 'r') as f: +with open("./tasks_config.json", "r") as f: tasks_config = json.load(f) scriptstring = """ @@ -9,15 +9,15 @@ import shifthappens.utils """ -print(tasks_config['tasks']) -for task in tasks_config['tasks']: - scriptstring += tasks_config['import_lines'][task] +print(tasks_config["tasks"]) +for task in tasks_config["tasks"]: + scriptstring += tasks_config["import_lines"][task] scriptstring += "\n" -scriptstring += tasks_config['import_lines'][tasks_config['model']] +scriptstring += tasks_config["import_lines"][tasks_config["model"]] -out_file_location = tasks_config['out_file_location'] -relative_data_folder = tasks_config['relative_data_folder'] +out_file_location = tasks_config["out_file_location"] +relative_data_folder = tasks_config["relative_data_folder"] scriptstring += f""" tasks = shifthappens.benchmark.get_task_registrations() @@ -30,6 +30,5 @@ with open(out_file_location, 'w') as outfile: outfile.write(results_string) """ -with open('./run_tasks.py', 'w') as run_script_file: +with open("./run_tasks.py", "w") as run_script_file: run_script_file.write(scriptstring) - diff --git a/shifthappens/task_data/task_metadata.py b/shifthappens/task_data/task_metadata.py index aaf6eb7..a6a2b97 100644 --- a/shifthappens/task_data/task_metadata.py +++ b/shifthappens/task_data/task_metadata.py @@ -28,9 +28,9 @@ def serialize_task_metadata(self) -> str: Serialize TaskMetadata object into json string. """ metadata_dict = { - 'name': self.name, - 'relative_data_folder': self.relative_data_folder, - 'standalone': self.standalone + "name": self.name, + "relative_data_folder": self.relative_data_folder, + "standalone": self.standalone, } return json.dumps(metadata_dict) @@ -41,10 +41,11 @@ def deserialize_task_metadata(metadata_str: str): """ metadata_dict = json.loads(metadata_str) metadata = TaskMetadata( - name=metadata_dict['name'], - relative_data_folder=metadata_dict['relative_data_folder'], - standalone=metadata_dict['standalone'] + name=metadata_dict["name"], + relative_data_folder=metadata_dict["relative_data_folder"], + standalone=metadata_dict["standalone"], ) return metadata + _TASK_METADATA_FIELD = "__task_metadata__" diff --git a/shifthappens/tasks/task_result.py b/shifthappens/tasks/task_result.py index cfa258a..90e01d6 100644 --- a/shifthappens/tasks/task_result.py +++ b/shifthappens/tasks/task_result.py @@ -71,25 +71,26 @@ def serialize_summary_metrics(self) -> str: """ return str({key.name: value for (key, value) in self.summary_metrics.items()}) - def serialize_task_result(self) -> str: """ Serializes TaskResult object into a string. """ result_dict = { - 'summary_metrics': self.serialize_summary_metrics(), - 'metrics': str(self._metrics) + "summary_metrics": self.serialize_summary_metrics(), + "metrics": str(self._metrics), } return str(result_dict) @staticmethod - def deserialize_summary_metrics(summary_metrics_str: str) -> Dict[Metric, Union[str, Tuple[str, ...]]]: + def deserialize_summary_metrics( + summary_metrics_str: str, + ) -> Dict[Metric, Union[str, Tuple[str, ...]]]: """ Deserializes valid string into summary_metrics. """ summary_metrics = eval(summary_metrics_str) result = {} - for (key, value) in summary_metrics.items(): + for key, value in summary_metrics.items(): result[Metric.__members__.get(key)] = value return result @@ -99,9 +100,8 @@ def deserialize_task_result(task_result_str: str): Deserializes valid string into a TaskResult object. """ result_dict = eval(task_result_str) - metrics = eval(result_dict['metrics']) - summary_metrics = TaskResult.deserialize_summary_metrics(result_dict['summary_metrics']) - return TaskResult( - summary_metrics=summary_metrics, - **metrics + metrics = eval(result_dict["metrics"]) + summary_metrics = TaskResult.deserialize_summary_metrics( + result_dict["summary_metrics"] ) + return TaskResult(summary_metrics=summary_metrics, **metrics) diff --git a/shifthappens/utils.py b/shifthappens/utils.py index 993f9e4..05d1627 100644 --- a/shifthappens/utils.py +++ b/shifthappens/utils.py @@ -140,18 +140,31 @@ def download_and_extract_archive( print(f"Extracting {archive} to {data_folder}") tv_utils.extract_archive(archive, data_folder, remove_finished) -def serialize_model_results(results: Dict[task_metadata.TaskMetadata, Union[TaskResult, None]]) -> str: +def serialize_model_results( + results: Dict[task_metadata.TaskMetadata, Union[TaskResult, None]] +) -> str: """ Converts evaluation results of a model into json objects. """ - return json.dumps({key.serialize_task_metadata(): value.serialize_task_result() for (key, value) in results.items() if value is not None}) - -def deserialize_model_results(results_str) -> Dict[task_metadata.TaskMetadata, TaskResult]: + return json.dumps( + { + key.serialize_task_metadata(): value.serialize_task_result() + for (key, value) in results.items() + if value is not None + } + ) + + +def deserialize_model_results( + results_str, +) -> Dict[task_metadata.TaskMetadata, TaskResult]: """ Converts json objects to a dictionary with (TaskMetadata, TaskResult) as (key, value) """ results_json_dict = json.loads(results_str) results = {} - for (key, value) in results_json_dict.items(): - results[task_metadata.TaskMetadata.deserialize_task_metadata(key)] = TaskResult.deserialize_task_result(value) + for key, value in results_json_dict.items(): + results[ + task_metadata.TaskMetadata.deserialize_task_metadata(key) + ] = TaskResult.deserialize_task_result(value) return results From 95101ad88769030b3bf436068fe76150451bb515 Mon Sep 17 00:00:00 2001 From: Nitin Date: Sat, 29 Apr 2023 15:44:47 +0200 Subject: [PATCH 8/8] Missing formatting changes --- shifthappens/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shifthappens/utils.py b/shifthappens/utils.py index 05d1627..528c646 100644 --- a/shifthappens/utils.py +++ b/shifthappens/utils.py @@ -140,6 +140,7 @@ def download_and_extract_archive( print(f"Extracting {archive} to {data_folder}") tv_utils.extract_archive(archive, data_folder, remove_finished) + def serialize_model_results( results: Dict[task_metadata.TaskMetadata, Union[TaskResult, None]] ) -> str: