Merge branch 'NKI-AI:main' into main

.dev/bump_date.py

@@ -0,0 +1,71 @@
+import re
+from datetime import datetime
+from typing import Callable, Dict
+
+
+def citation_regex(new_date, content):
+    """
+    Replaces the date in the CITATION.cff file with the new date
+    """
+    return re.sub(r"date-released: \d{4}-\d{2}-\d{2}", f"date-released: {new_date}", content)
+
+
+readme_regexes = {
+    "bibtex_year": lambda year, content: re.sub(r"\{\d{4}\}", f"{{{year}}}", content),  # {2023} in bibtex
+    "bibtex_month": lambda month, content: re.sub(r"\{\d{1,2}\}", f"{{{month}}}", content),  # {5} or {12} in bibtex
+    "plain_bib_year": lambda year, content: re.sub(r"(\d{4})", str(year), content),  # (2023) in the plain bibliography
+}
+
+
+def bump_citation_file(
+    new_date: str,
+    filename: str = "CITATION.cff",
+    regex_function: Callable = citation_regex,
+) -> None:
+    """
+    Runs a specific regex function on the defined citation file
+    """
+    with open(filename, "r") as f:
+        content = f.read()
+
+    with open(filename, "w") as f:
+        content = regex_function(new_date, content)
+        f.write(content)
+
+
+def bump_readme_file(
+    year: str,
+    month: str,
+    regex_functions: Dict[str, Callable] = readme_regexes,
+    filename: str = "README.md",
+) -> None:
+    """
+    Runs the required regex functions on the README file
+    """
+    with open(filename, "r") as f:
+        content = f.read()
+
+    with open(filename, "w") as f:
+        content = regex_functions["bibtex_year"](year, content)
+        content = regex_functions["bibtex_month"](month, content)
+        content = regex_functions["plain_bib_year"](year, content)
+        f.write(content)
+
+
+def bump_date() -> None:
+    """
+    A developer function to bump the date when a version is bumped.
+
+    Run after bump2version is run, since bump2version doesn't allow updating
+    the date in e.g. README.md and CITATION.cff.
+    """
+    new_date = datetime.now().strftime("%Y-%m-%d")
+    year = str(datetime.now().year)
+    month = str(datetime.now().month)
+
+    bump_citation_file(new_date=new_date)
+    bump_readme_file(year=year, month=month)
+
+
+if __name__ == "__main__":
+    bump_date()

.github/workflows/tox.yml

@@ -28,4 +28,3 @@ jobs:
         pip install tox tox-gh-actions
     - name: Test with tox
       run: tox
-

.pre-commit-config.yaml

@@ -1,60 +1,51 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v3.2.0
     hooks:
-    - id: check-ast
-    - id: check-case-conflict
-    - id: check-executables-have-shebangs
-    - id: check-json
-    - id: check-merge-conflict
-    - id: check-symlinks
-    - id: check-vcs-permalinks
-    - id: check-xml
-    - id: check-yaml
-      args: [--unsafe]
-    - id: end-of-file-fixer
-    - id: fix-encoding-pragma
-      args: [ '--pragma=# coding=utf-8' ]
-    - id: requirements-txt-fixer
-    - id: check-added-large-files
-      args: [ '--maxkb=500' ]
-    - id: sort-simple-yaml
-    - id: trailing-whitespace
-    - id: no-commit-to-branch
-      args: [--branch, main]
-    - id: forbid-new-submodules
-- repo: https://github.com/pycqa/isort
-  rev: 5.8.0
-  hooks:
-    - id: isort
-      name: isort (python)
-    - id: isort
-      name: isort (cython)
-      types: [cython]
-    - id: isort
-      name: isort (pyi)
-      types: [pyi]
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
 -   repo: https://github.com/psf/black
-    rev: 21.6b0
+    rev: 23.7.0
     hooks:
     -   id: black
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.902
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    # Ignore the configuration files
     hooks:
-    - id: mypy
-      additional_dependencies: [types-requests]
--   repo: https://github.com/Yelp/detect-secrets
-    rev: v1.1.0
+    -   id: flake8
+        exclude: ^docs/
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
-    -   id: detect-secrets
-        args: ['--baseline', '.secrets.baseline']
--   repo: local
+      - id: isort
+        name: isort (python)
+-   repo: local # Use pylint from local environment as it requires to import packages
     hooks:
-    -   id: pylint
-        name: pylint
-        entry: pylint
+    -   id: mypy
+        name: mypy
+        entry: mypy
         language: system
         types: [python]
-        args:
-            - dlup
-            - --errors-only
+        args: ["--strict"]
+        files: ^dlup/
+-   repo: https://github.com/Yelp/detect-secrets
+    rev: v1.4.0
+    hooks:
+    -   id: detect-secrets
+- repo: local  # Use pylint from local environment as it requires to import packages
+  hooks:
+    - id: pylint
+      name: pylint
+      entry: pylint
+      language: system
+      types: [python]
+      args:
+        [
+          "-rn", # Only display messages
+          "-sn", # Don't display the score
+          "--errors-only"  # Only show the errors
+        ]

CITATION.cff

@@ -0,0 +1,21 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+- family-names: "Teuwen"
+  given-names: "Jonas"
+  orcid: "https://orcid.org/0000-0002-1825-1428"
+- family-names: "Romor"
+  given-names: "Leonardo"
+- family-names: "Pai"
+  given-names: "Ajey"
+  orcid: "https://orcid.org/0009-0003-3970-9236"
+- family-names: "Schirris"
+  given-names: "Yoni"
+  orcid: "https://orcid.org/0000-0003-0217-8737"
+- family-names: "Marcus"
+  given-names: "Eric"
+  orchid: "https://orcid.org/0000-0002-3375-6248"
+title: "DLUP: Deep Learning Utilities for Pathology"
+version: 0.3.31
+date-released: 2023-10-17
+url: "https://github.com/nki-ai/dlup"

CONTRIBUTING.rst

@@ -62,6 +62,8 @@ Ready to contribute? Here's how to set up `dlup` for local development.
    - Install `pre-commit`_.
    - Install pre-commit hooks: :code:`pre-commit install`.
 
+Now the pre-commit hooks will run before committing, but you can also run `pre-commit run --all` explicitly.
+
 5. Install your local copy into a virtual environment:
 
 .. code-block:: console
@@ -127,6 +129,7 @@ Make sure all your changes are committed. Then run:
 .. code-block:: console
 
     $ bump2version patch # possible: major / minor / patch
+    $ python .dev/bump_date.py
     $ git push
 
 

README.md

@@ -21,3 +21,23 @@ The package can be installed using `python -m pip install dlup`.
 
 ## Used by
 - [ahcore](https://github.com/NKI-AI/ahcore.git): a pytorch lightning based-library for computational pathology
+
+## Citing DLUP
+If you use DLUP in your own research, please use the following BiBTeX entry:
+
+```
+@software{dlup,
+  author = {Teuwen, J., Romor, L., Pai, A., Schirris, Y., Marcus, E.},
+  month = {10},
+  title = {{DLUP: Deep Learning Utilities for Pathology}},
+  url = {https://github.com/nki-ai/dlup},
+  version = {0.3.31},
+  year = {2023}
+}
+```
+
+or the following plain bibliography:
+
+```
+Teuwen, J., Romor, L., Pai, A., Schirris, Y., Marcus E. (2023). DLUP: Deep Learning Utilities for Pathology (Version 0.3.31) [Computer software]. https://github.com/nki-ai/dlup
+```

dlup/__init__.py

@@ -6,6 +6,6 @@
 
 __author__ = """dlup contributors"""
 __email__ = "[email protected]"
-__version__ = "0.3.29"
+__version__ = "0.3.31"
 
 __all__ = ("SlideImage", "RegionView", "UnsupportedSlideError", "BoundaryMode")

dlup/_image.py

@@ -226,10 +226,13 @@ def read_region(
         level_size = np.array(self.get_scaled_size(scaling))
 
         if (size < 0).any():
-            raise ValueError("Size values must be greater than zero.")
+            raise ValueError(f"Size values must be greater than zero. Got {size}")
 
         if ((location < 0) | ((location + size) > level_size)).any():
-            raise ValueError("Requested region is outside level boundaries.")
+            raise ValueError(
+                f"Requested region is outside level boundaries. "
+                f"{location.tolist()} + {size.tolist()} (={(location + size).tolist()}) > {level_size.tolist()}."
+            )
 
         native_level = owsi.get_best_level_for_downsample(1 / scaling)
         native_level_size = owsi.level_dimensions[native_level]

dlup/annotations.py

@@ -671,8 +671,8 @@ def from_darwin_json(cls, darwin_json: PathLike, scaling: float | None = None) -
 
         # Now we can make SingleAnnotationWrapper annotations
         output = []
-        for an_cls, annotations in all_annotations.items():
-            output.append(SingleAnnotationWrapper(a_cls=an_cls, annotation=annotations))
+        for an_cls, _annotation in all_annotations.items():
+            output.append(SingleAnnotationWrapper(a_cls=an_cls, annotation=_annotation))
         return cls(output, sorting=AnnotationSorting.NONE)
 
     def __getitem__(self, a_cls: AnnotationClass) -> SingleAnnotationWrapper:

dlup/cli/mask.py

@@ -2,7 +2,7 @@
 """CLI utilities to handle masks"""
 import argparse
 import json
-from typing import Any, cast
+from typing import cast
 
 import shapely
 

dlup/data/dataset.py

@@ -148,7 +148,7 @@ def index_to_dataset(self, idx: int) -> tuple[Dataset[T_co], int]:
         """
         if idx < 0:
             if -idx > len(self):
-                raise ValueError("absolute value of index should not exceed dataset length")
+                raise ValueError("Absolute value of index should not exceed dataset length")
             idx = len(self) + idx
         dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx)
         sample_idx = idx if dataset_idx == 0 else idx - self.cumulative_sizes[dataset_idx - 1]
@@ -568,6 +568,10 @@ def _process_tile_sample(self, region_data: TileSample) -> RegionFromWsiDatasetS
 
         return output
 
+    def __iter__(self) -> Iterator[RegionFromWsiDatasetSample]:
+        for i in range(len(self)):
+            yield self[i]
+
 
 def parse_rois(rois: list[ROIType] | None, image_size: tuple[int, int], scaling: float = 1.0) -> list[ROIType]:
     if rois is None:

dlup/data/transforms.py

@@ -157,6 +157,43 @@ def __call__(self, sample: TileSample) -> TileSampleWithAnnotationData:
         return output
 
 
+def rename_labels(annotations: Iterable[_AnnotationsTypes], remap_labels: dict[str, str]) -> list[_AnnotationsTypes]:
+    """
+    Rename the labels in the annotations.
+
+    Parameters
+    ----------
+    annotations: Iterable[_AnnotationsTypes]
+        The annotations
+    remap_labels: dict[str, str]
+        The renaming table
+
+    Returns
+    -------
+    list[_AnnotationsTypes]
+    """
+    output_annotations = []
+    for annotation in annotations:
+        label = annotation.label
+        if label not in remap_labels:
+            output_annotations.append(annotation)
+            continue
+
+        if annotation.a_cls.a_cls == AnnotationType.BOX:
+            a_cls = AnnotationClass(label=remap_labels[label], a_cls=AnnotationType.BOX)
+            output_annotations.append(dlup.annotations.Polygon(annotation, a_cls=a_cls))
+        elif annotation.a_cls.a_cls == AnnotationType.POLYGON:
+            a_cls = AnnotationClass(label=remap_labels[label], a_cls=AnnotationType.POLYGON)
+            output_annotations.append(dlup.annotations.Polygon(annotation, a_cls=a_cls))
+        elif annotation.a_cls.a_cls == AnnotationType.POINT:
+            a_cls = AnnotationClass(label=remap_labels[label], a_cls=AnnotationType.POINT)
+            output_annotations.append(dlup.annotations.Point(annotation, a_cls=a_cls))
+        else:
+            raise AnnotationError(f"Unsupported annotation type {annotation.a_cls.a_cls}")
+
+    return output_annotations
+
+
 class RenameLabels:
     """Remap the label names"""
 
@@ -175,26 +212,7 @@ def __call__(self, sample: TileSample) -> TileSample:
         if not _annotations:
             raise ValueError("No annotations found to rename.")
 
-        output_annotations = []
-        for annotation in _annotations:
-            label = annotation.label
-            if label not in self._remap_labels:
-                output_annotations.append(annotation)
-                continue
-
-            if annotation.a_cls.a_cls == AnnotationType.BOX:
-                a_cls = AnnotationClass(label=self._remap_labels[label], a_cls=AnnotationType.BOX)
-                output_annotations.append(dlup.annotations.Polygon(annotation, a_cls=a_cls))
-            elif annotation.a_cls.a_cls == AnnotationType.POLYGON:
-                a_cls = AnnotationClass(label=self._remap_labels[label], a_cls=AnnotationType.POLYGON)
-                output_annotations.append(dlup.annotations.Polygon(annotation, a_cls=a_cls))
-            elif annotation.a_cls.a_cls == AnnotationType.POINT:
-                a_cls = AnnotationClass(label=self._remap_labels[label], a_cls=AnnotationType.POINT)
-                output_annotations.append(dlup.annotations.Point(annotation, a_cls=a_cls))
-            else:
-                raise AnnotationError(f"Unsupported annotation type {annotation.a_cls.a_cls}")
-
-        sample["annotations"] = output_annotations
+        sample["annotations"] = rename_labels(_annotations, self._remap_labels)
         return sample
 
 

dlup/tiling.py

@@ -9,8 +9,6 @@
 import numpy as np
 import numpy.typing as npt
 
-from dlup.types import GenericNumberArray
-
 _GenericNumber = Union[int, float]
 _GenericNumberArray = Union[npt.NDArray[np.int_ | np.float_], Sequence[_GenericNumber]]
 

docker/jupyter_notebook_config.py

@@ -1,9 +1,9 @@
 # coding=utf-8
 import os
 
-from IPython.lib import passwd
+from IPython.lib import passwd  # pylint: disable=no-name-in-module
 
-c = get_config()  # type: ignore # pylint: disable=undefined-variable
+c = get_config()  # type: ignore # pylint: disable=undefined-variable # noqa
 c.NotebookApp.ip = "0.0.0.0"
 c.NotebookApp.port = int(os.getenv("PORT", 8888))
 c.NotebookApp.open_browser = False