Panda SSL tile selection and binary evaluation (#635)

* panda tile selection and binary label * changelog * PR comments * flake8 errors * remove reset_index * update pillow and ipython Co-authored-by: t-hsharma <[email protected]>
microsoft · Jan 24, 2022 · 884e3fd · 884e3fd
1 parent 56ee8b6
commit 884e3fd
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -40,6 +40,7 @@ jobs that run in AzureML.
 - ([#616](https://github.com/microsoft/InnerEye-DeepLearning/pull/616)) Add more histopathology configs and tests
 - ([#621](https://github.com/microsoft/InnerEye-DeepLearning/pull/621)) Add WSI preprocessing functions and enable tiling more generic slide datasets
 - ([#634](https://github.com/microsoft/InnerEye-DeepLearning/pull/634)) Add WSI heatmaps and thumbnails to standard test outputs
+- ([#635](https://github.com/microsoft/InnerEye-DeepLearning/pull/635)) Add tile selection and binary label for online evaluation of PANDA SSL
 
 ### Changed
 - ([#588](https://github.com/microsoft/InnerEye-DeepLearning/pull/588)) Replace SciPy with PIL.PngImagePlugin.PngImageFile to load png files.

diff --git a/InnerEye/ML/Histopathology/datasets/panda_tiles_dataset.py b/InnerEye/ML/Histopathology/datasets/panda_tiles_dataset.py
@@ -35,36 +35,45 @@ class PandaTilesDataset(TilesDataset):
     def __init__(self,
                  root: Path,
                  dataset_csv: Optional[Union[str, Path]] = None,
-                 dataset_df: Optional[pd.DataFrame] = None) -> None:
+                 dataset_df: Optional[pd.DataFrame] = None,
+                 occupancy_threshold: Optional[float] = None) -> None:
         super().__init__(root=Path(root) / self._RELATIVE_ROOT_FOLDER,
                          dataset_csv=dataset_csv,
                          dataset_df=dataset_df,
                          train=None)
-
+        if occupancy_threshold is not None:
+            dataset_df_filtered = self.dataset_df.loc[self.dataset_df['occupancy'] > occupancy_threshold]  # type: ignore
+            self.dataset_df = dataset_df_filtered
 
 class PandaTilesDatasetReturnImageLabel(VisionDataset):
     """
     Any dataset used in SSL needs to return a tuple where the first element is the image and the second is a
     class label.
     """
+    occupancy_threshold = 0
+
     def __init__(self,
                  root: Path,
                  dataset_csv: Optional[Union[str, Path]] = None,
                  dataset_df: Optional[pd.DataFrame] = None,
                  transform: Optional[Callable] = None,
                  **kwargs: Any) -> None:
         super().__init__(root=root, transform=transform)
+
         self.base_dataset = PandaTilesDataset(root=root,
                                               dataset_csv=dataset_csv,
-                                              dataset_df=dataset_df)
+                                              dataset_df=dataset_df,
+                                              occupancy_threshold=self.occupancy_threshold)
 
     def __getitem__(self, index: int) -> Tuple:  # type: ignore
         sample = self.base_dataset[index]
         # TODO change to a meaningful evaluation
         image = load_pil_image(sample[self.base_dataset.IMAGE_COLUMN])
         if self.transform:
             image = self.transform(image)
-        return image, 1
+        # get binary label
+        label = 0 if sample[self.base_dataset.LABEL_COLUMN] == 0 else 1
+        return image, label
 
     def __len__(self) -> int:
         return len(self.base_dataset)

diff --git a/environment.yml b/environment.yml
@@ -27,6 +27,7 @@ dependencies:
       - gitpython==3.1.7
       - gputil==1.4.0
       - h5py==2.10.0
+      - ipython==7.31.1
       - InnerEye-DICOM-RT==1.0.1
       - joblib==0.16.0
       - jupyter==1.0.0
@@ -44,7 +45,7 @@ dependencies:
       - pandas==1.1.0
       - papermill==2.2.2
       - param==1.9.3
-      - pillow==8.3.2
+      - pillow==9.0.0
       - psutil==5.7.2
       - pydicom==2.0.0
       - pyflakes==2.2.0