Add a tool to help conduct experiments

src/otx/algorithms/classification/adapters/openvino/task.py

@@ -176,6 +176,12 @@ def __init__(self, task_environment: TaskEnvironment):
         self.inferencer = self.load_inferencer()
         template_file_path = self.task_environment.model_template.model_template_path
         self._base_dir = os.path.abspath(os.path.dirname(template_file_path))
+        self._avg_time_per_image: Optional[float] = None
+
+    @property
+    def avg_time_per_image(self) -> Optional[float]:
+        """Average inference time per image."""
+        return self._avg_time_per_image
 
     def load_inferencer(self) -> ClassificationOpenVINOInferencer:
         """load_inferencer function of ClassificationOpenVINOTask."""
@@ -270,7 +276,8 @@ def add_prediction(id: int, predicted_scene: AnnotationSceneEntity, aux_data: tu
 
         self.inferencer.await_all()
 
-        logger.info(f"Avg time per image: {total_time/len(dataset)} secs")
+        self._avg_time_per_image = total_time / len(dataset)
+        logger.info(f"Avg time per image: {self._avg_time_per_image} secs")
         logger.info(f"Total time: {total_time} secs")
         logger.info("Classification OpenVINO inference completed")
 

src/otx/algorithms/detection/adapters/openvino/task.py

@@ -387,13 +387,19 @@ def __init__(self, task_environment: TaskEnvironment):
         self.confidence_threshold: float = 0.0
         self.config = self.load_config()
         self.inferencer = self.load_inferencer()
+        self._avg_time_per_image: Optional[float] = None
         logger.info("OpenVINO task initialization completed")
 
     @property
     def hparams(self):
         """Hparams of OpenVINO Detection Task."""
         return self.task_environment.get_hyper_parameters(DetectionConfig)
 
+    @property
+    def avg_time_per_image(self) -> Optional[float]:
+        """Average inference time per image."""
+        return self._avg_time_per_image
+
     def load_config(self) -> ADDict:
         """Load configurable parameters from model adapter.
 
@@ -557,7 +563,8 @@ def add_prediction(id: int, predicted_scene: AnnotationSceneEntity, aux_data: tu
 
         self.inferencer.await_all()
 
-        logger.info(f"Avg time per image: {total_time/len(dataset)} secs")
+        self._avg_time_per_image = total_time / len(dataset)
+        logger.info(f"Avg time per image: {self._avg_time_per_image} secs")
         logger.info(f"Total time: {total_time} secs")
         logger.info("OpenVINO inference completed")
         return dataset

src/otx/algorithms/segmentation/adapters/openvino/task.py

@@ -162,6 +162,7 @@ def __init__(self, task_environment: TaskEnvironment):
         self.model = self.task_environment.model
         self.model_name = self.task_environment.model_template.model_template_id
         self.inferencer = self.load_inferencer()
+        self._avg_time_per_image: Optional[float] = None
 
         labels = task_environment.get_labels(include_empty=False)
         self._label_dictionary = dict(enumerate(labels, 1))
@@ -173,6 +174,11 @@ def hparams(self):
         """Hparams of OpenVINO Segmentation Task."""
         return self.task_environment.get_hyper_parameters(SegmentationConfig)
 
+    @property
+    def avg_time_per_image(self) -> Optional[float]:
+        """Average inference time per image."""
+        return self._avg_time_per_image
+
     def load_inferencer(self) -> OpenVINOSegmentationInferencer:
         """load_inferencer function of OpenVINO Segmentation Task."""
         if self.model is None:
@@ -248,7 +254,8 @@ def add_prediction(
 
         self.inferencer.await_all()
 
-        logger.info(f"Avg time per image: {total_time/len(dataset)} secs")
+        self._avg_time_per_image = total_time / len(dataset)
+        logger.info(f"Avg time per image: {self._avg_time_per_image} secs")
         logger.info(f"Total time: {total_time} secs")
         logger.info("Segmentation OpenVINO inference completed")
 

src/otx/algorithms/visual_prompting/tasks/openvino.py

@@ -258,6 +258,7 @@ def __init__(self, task_environment: TaskEnvironment) -> None:
         self.model = self.task_environment.model
         self.model_name = self.task_environment.model_template.model_template_id
         self.inferencer = self.load_inferencer()
+        self._avg_time_per_image: Optional[float] = None
 
         labels = task_environment.get_labels(include_empty=False)
         self._label_dictionary = dict(enumerate(labels, 1))
@@ -270,6 +271,11 @@ def hparams(self):
         """Hparams of OpenVINO Visual Prompting Task."""
         return self.task_environment.get_hyper_parameters(VisualPromptingBaseConfig)
 
+    @property
+    def avg_time_per_image(self) -> Optional[float]:
+        """Average inference time per image."""
+        return self._avg_time_per_image
+
     def load_inferencer(self) -> OpenVINOVisualPromptingInferencer:
         """Load OpenVINO Visual Prompting Inferencer."""
         if self.model is None:
@@ -328,7 +334,8 @@ def add_prediction(id: int, annotations: List[Annotation]):
 
         self.inferencer.await_all()
 
-        logger.info(f"Avg time per image: {total_time/len(dataset)} secs")
+        self._avg_time_per_image = total_time / len(dataset)
+        logger.info(f"Avg time per image: {self._avg_time_per_image} secs")
         logger.info(f"Total time: {total_time} secs")
         logger.info("Visual Prompting OpenVINO inference completed")
 

src/otx/cli/tools/eval.py

@@ -156,11 +156,11 @@ def main():
     print(resultset.performance)
 
     output_path = Path(args.output) if args.output else config_manager.output_path
+    performance = {resultset.performance.score.name: resultset.performance.score.value}
+    if hasattr(task, "avg_time_per_image"):
+        performance["avg_time_per_image"] = task.avg_time_per_image
     with open(output_path / "performance.json", "w", encoding="UTF-8") as write_file:
-        json.dump(
-            {resultset.performance.score.name: resultset.performance.score.value},
-            write_file,
-        )
+        json.dump(performance, write_file)
 
     return dict(retcode=0, template=template.name)