From c45f4502cf87491612952f548d6ae3589717cf19 Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 18 Aug 2020 17:01:47 +0200 Subject: [PATCH 01/39] Open images for a mask via the array link --- ome_zarr.py | 52 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/ome_zarr.py b/ome_zarr.py index 2c327677..94b17b7a 100644 --- a/ome_zarr.py +++ b/ome_zarr.py @@ -14,6 +14,7 @@ import json import requests import dask.array as da +import posixpath import warnings from dask.diagnostics import ProgressBar @@ -128,7 +129,10 @@ def to_rgba(self, v): """Get rgba (0-1) e.g. (1, 0.5, 0, 1) from integer""" return [x / 255 for x in v.to_bytes(4, signed=True, byteorder="big")] - def reader_function(self, path: Optional[PathLike]) -> Optional[List[LayerData]]: + def reader_function(self, + path: Optional[PathLike], + recurse: bool = True, + ) -> Optional[List[LayerData]]: """Take a path or list of paths and return a list of LayerData tuples.""" if isinstance(path, list): @@ -139,23 +143,48 @@ def reader_function(self, path: Optional[PathLike]) -> Optional[List[LayerData]] LOGGER.debug(f"treating {path} as ome-zarr") layers = [self.load_ome_zarr()] # If the Image contains labels... - if self.has_ome_labels(): + if recurse and self.has_ome_labels(): label_path = os.path.join(self.zarr_path, "labels") # Create a new OME Zarr Reader to load labels - labels = self.__class__(label_path).reader_function(None) + labels = self.__class__(label_path).reader_function( + None, recurse=False) if labels: layers.extend(labels) return layers + elif self.is_ome_label(): + LOGGER.debug(f"treating {path} as labels") + layers = self.load_ome_labels() + rv = [] + try: + for layer in layers: + metadata = layer[1].get("metadata", {}) + path = metadata.get("path", None) + array = metadata.get("image", {}).get("array", None) + if recurse and path and array: + # This is an ome mask, load the image + parent = posixpath.normpath(f"{path}/{array}") + LOGGER.debug(f"delegating to parent image: {parent}") + # Create a new OME Zarr Reader to load labels + replace = self.__class__(parent).reader_function( + None, recurse=False) + for r in replace: + r[1]["visible"] = False + rv.extend(replace) + layer[1]["visible"] = True + rv.append(layer) + return rv + except Exception as e: + LOGGER.error(e) + return [] + + # TODO: might also be an individiaul mask + elif self.zarray: LOGGER.debug(f"treating {path} as raw zarr") data = da.from_zarr(f"{self.zarr_path}") return [(data,)] - elif self.is_ome_label(): - LOGGER.debug(f"treating {path} as labels") - return self.load_ome_labels() - else: LOGGER.debug(f"ignoring {path}") return None @@ -290,7 +319,14 @@ def load_ome_labels(self): labels.append( ( data[:, n, :, :, :], - {"visible": False, "name": name, "color": colors}, + {"visible": False, + "name": name, + "color": colors, + "metadata": { + "image": label_attrs.get("image", {}), + "path": label_path, + }, + }, "labels", ) ) From 47d8563190a58569cf46326e4b65043fab33e826 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 19 Aug 2020 09:01:48 +0200 Subject: [PATCH 02/39] Move python code to a module --- ome_zarr/__init__.py | 0 ome_zarr_cli.py => ome_zarr/cli.py | 4 +- ome_zarr/napari.py | 45 ++++++++++ ome_zarr.py => ome_zarr/reader.py | 139 ++++------------------------- ome_zarr/utils.py | 88 ++++++++++++++++++ setup.py | 6 +- tests/test_ome_zarr.py | 3 +- 7 files changed, 157 insertions(+), 128 deletions(-) create mode 100644 ome_zarr/__init__.py rename ome_zarr_cli.py => ome_zarr/cli.py (94%) create mode 100644 ome_zarr/napari.py rename ome_zarr.py => ome_zarr/reader.py (73%) create mode 100644 ome_zarr/utils.py diff --git a/ome_zarr/__init__.py b/ome_zarr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ome_zarr_cli.py b/ome_zarr/cli.py similarity index 94% rename from ome_zarr_cli.py rename to ome_zarr/cli.py index 6dd32789..d47deeab 100755 --- a/ome_zarr_cli.py +++ b/ome_zarr/cli.py @@ -3,8 +3,8 @@ import argparse import logging -from ome_zarr import info as zarr_info -from ome_zarr import download as zarr_download +from .utils import info as zarr_info +from .utils import download as zarr_download def config_logging(loglevel, args): diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py new file mode 100644 index 00000000..06285ffb --- /dev/null +++ b/ome_zarr/napari.py @@ -0,0 +1,45 @@ +""" +This module is a napari plugin. + +It implements the ``napari_get_reader`` hook specification, (to create +a reader plugin). +""" + + +try: + from napari_plugin_engine import napari_hook_implementation +except ImportError: + + def napari_hook_implementation(func, *args, **kwargs): + return func + + +import logging + +# for optional type hints only, otherwise you can delete/ignore this stuff +from typing import List, Optional, Union, Any, Tuple, Dict, Callable + +from .utils import parse_url + +LOGGER = logging.getLogger("ome_zarr.napari") + +LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] +PathLike = Union[str, List[str]] +ReaderFunction = Callable[[PathLike], List[LayerData]] +# END type hint stuff. + + +@napari_hook_implementation +def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: + """ + Returns a reader for supported paths that include IDR ID + + - URL of the form: https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/ID.zarr/ + """ + if isinstance(path, list): + path = path[0] + instance = parse_url(path) + if instance is not None and instance.is_zarr(): + return instance.get_reader_function() + # Ignoring this path + return None diff --git a/ome_zarr.py b/ome_zarr/reader.py similarity index 73% rename from ome_zarr.py rename to ome_zarr/reader.py index 94b17b7a..367c6599 100644 --- a/ome_zarr.py +++ b/ome_zarr/reader.py @@ -1,14 +1,5 @@ """ -This module is a napari plugin. - -It implements the ``napari_get_reader`` hook specification, (to create -a reader plugin). - -Type annotations here are OPTIONAL! -If you don't care to annotate the return types of your functions -your plugin doesn't need to import, or even depend on napari at all! - -Replace code below accordingly. +Reading logic for ome-zarr """ import os import json @@ -17,63 +8,20 @@ import posixpath import warnings -from dask.diagnostics import ProgressBar from vispy.color import Colormap -from urllib.parse import urlparse - - -try: - from napari_plugin_engine import napari_hook_implementation -except ImportError: - - def napari_hook_implementation(func, *args, **kwargs): - return func - - import logging # for optional type hints only, otherwise you can delete/ignore this stuff -from typing import List, Optional, Union, Any, Tuple, Dict, Callable - -LOGGER = logging.getLogger("ome_zarr") +from typing import List, Optional, Union, Any, Tuple, Dict, cast +LOGGER = logging.getLogger("ome_zarr.reader") LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] PathLike = Union[str, List[str]] -ReaderFunction = Callable[[PathLike], List[LayerData]] # END type hint stuff. -@napari_hook_implementation -def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: - """ - Returns a reader for supported paths that include IDR ID - - - URL of the form: https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/ID.zarr/ - """ - if isinstance(path, list): - path = path[0] - instance = parse_url(path) - if instance is not None and instance.is_zarr(): - return instance.get_reader_function() - # Ignoring this path - return None - - -def parse_url(path): - # Check is path is local directory first - if os.path.isdir(path): - return LocalZarr(path) - else: - result = urlparse(path) - if result.scheme in ("", "file"): - # Strips 'file://' if necessary - return LocalZarr(result.path) - else: - return RemoteZarr(path) - - class BaseZarr: def __init__(self, path): self.zarr_path = path.endswith("/") and path or f"{path}/" @@ -129,10 +77,9 @@ def to_rgba(self, v): """Get rgba (0-1) e.g. (1, 0.5, 0, 1) from integer""" return [x / 255 for x in v.to_bytes(4, signed=True, byteorder="big")] - def reader_function(self, - path: Optional[PathLike], - recurse: bool = True, - ) -> Optional[List[LayerData]]: + def reader_function( + self, path: Optional[PathLike], recurse: bool = True, + ) -> Optional[List[LayerData]]: """Take a path or list of paths and return a list of LayerData tuples.""" if isinstance(path, list): @@ -146,8 +93,7 @@ def reader_function(self, if recurse and self.has_ome_labels(): label_path = os.path.join(self.zarr_path, "labels") # Create a new OME Zarr Reader to load labels - labels = self.__class__(label_path).reader_function( - None, recurse=False) + labels = self.__class__(label_path).reader_function(None, recurse=False) if labels: layers.extend(labels) return layers @@ -167,7 +113,8 @@ def reader_function(self, LOGGER.debug(f"delegating to parent image: {parent}") # Create a new OME Zarr Reader to load labels replace = self.__class__(parent).reader_function( - None, recurse=False) + None, recurse=False + ) for r in replace: r[1]["visible"] = False rv.extend(replace) @@ -319,13 +266,14 @@ def load_ome_labels(self): labels.append( ( data[:, n, :, :, :], - {"visible": False, - "name": name, - "color": colors, - "metadata": { - "image": label_attrs.get("image", {}), - "path": label_path, - }, + { + "visible": False, + "name": name, + "color": colors, + "metadata": { + "image": label_attrs.get("image", {}), + "path": label_path, + }, }, "labels", ) @@ -360,56 +308,3 @@ def get_json(self, subpath): except Exception: LOGGER.error(f"({rsp.status_code}): {rsp.text}") return {} - - -def info(path): - """ - print information about the ome-zarr fileset - """ - zarr = parse_url(path) - if not zarr.is_ome_zarr(): - print(f"not an ome-zarr: {zarr}") - return - reader = zarr.get_reader_function() - data = reader(path) - LOGGER.debug(data) - - -def download(path, output_dir=".", zarr_name=""): - """ - download zarr from URL - """ - omezarr = parse_url(path) - if not omezarr.is_ome_zarr(): - print(f"not an ome-zarr: {path}") - return - - image_id = omezarr.image_data.get("id", "unknown") - LOGGER.info("image_id %s", image_id) - if not zarr_name: - zarr_name = f"{image_id}.zarr" - - try: - datasets = [x["path"] for x in omezarr.root_attrs["multiscales"][0]["datasets"]] - except KeyError: - datasets = ["0"] - LOGGER.info("datasets %s", datasets) - resolutions = [da.from_zarr(path, component=str(i)) for i in datasets] - # levels = list(range(len(resolutions))) - - target_dir = os.path.join(output_dir, f"{zarr_name}") - if os.path.exists(target_dir): - print(f"{target_dir} already exists!") - return - print(f"downloading to {target_dir}") - - pbar = ProgressBar() - for dataset, data in reversed(list(zip(datasets, resolutions))): - LOGGER.info(f"resolution {dataset}...") - with pbar: - data.to_zarr(os.path.join(target_dir, dataset)) - - with open(os.path.join(target_dir, ".zgroup"), "w") as f: - f.write(json.dumps(omezarr.zgroup)) - with open(os.path.join(target_dir, ".zattrs"), "w") as f: - f.write(json.dumps(omezarr.root_attrs)) diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py new file mode 100644 index 00000000..c0583cec --- /dev/null +++ b/ome_zarr/utils.py @@ -0,0 +1,88 @@ +""" +Utility methods for ome_zarr access +""" + +import os +import json +import dask.array as da + +from dask.diagnostics import ProgressBar + +from urllib.parse import urlparse + +import logging + +from .reader import ( + LocalZarr, + RemoteZarr, +) + + +LOGGER = logging.getLogger("ome_zarr.utils") + + +def parse_url(path): + # Check is path is local directory first + if os.path.isdir(path): + return LocalZarr(path) + else: + result = urlparse(path) + if result.scheme in ("", "file"): + # Strips 'file://' if necessary + return LocalZarr(result.path) + else: + return RemoteZarr(path) + + +def info(path): + """ + print information about the ome-zarr fileset + """ + zarr = parse_url(path) + if not zarr.is_ome_zarr(): + print(f"not an ome-zarr: {zarr}") + return + reader = zarr.get_reader_function() + data = reader(path) + LOGGER.debug(data) + + +def download(path, output_dir=".", zarr_name=""): + """ + download zarr from URL + """ + omezarr = parse_url(path) + if not omezarr.is_ome_zarr(): + print(f"not an ome-zarr: {path}") + return + + image_id = omezarr.image_data.get("id", "unknown") + LOGGER.info("image_id %s", image_id) + if not zarr_name: + zarr_name = f"{image_id}.zarr" + + try: + datasets = omezarr.root_attrs["multiscales"][0]["datasets"] + datasets = [x["path"] for x in datasets] + except KeyError: + datasets = ["0"] + LOGGER.info("datasets %s", datasets) + resolutions = [da.from_zarr(path, component=str(i)) for i in datasets] + # levels = list(range(len(resolutions))) + + target_dir = os.path.join(output_dir, f"{zarr_name}") + if os.path.exists(target_dir): + print(f"{target_dir} already exists!") + return + print(f"downloading to {target_dir}") + + pbar = ProgressBar() + for dataset, data in reversed(list(zip(datasets, resolutions))): + LOGGER.info(f"resolution {dataset}...") + with pbar: + data.to_zarr(os.path.join(target_dir, dataset)) + + with open(os.path.join(target_dir, ".zgroup"), "w") as f: + f.write(json.dumps(omezarr.zgroup)) + with open(os.path.join(target_dir, ".zattrs"), "w") as f: + f.write(json.dumps(omezarr.root_attrs)) diff --git a/setup.py b/setup.py index 7b7282fd..282d8fd6 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def read(fname): url="https://github.com/ome/ome-zarr-py", description="Implementation of images in Zarr files.", long_description=read("README.rst"), - py_modules=["ome_zarr", "ome_zarr_cli"], + py_modules=["ome_zarr"], python_requires=">=3.6", install_requires=install_requires, classifiers=[ @@ -43,8 +43,8 @@ def read(fname): "License :: OSI Approved :: BSD License", ], entry_points={ - "console_scripts": ["ome_zarr = ome_zarr_cli:main"], - "napari.plugin": ["ome_zarr = ome_zarr"], + "console_scripts": ["ome_zarr = ome_zarr.cli:main"], + "napari.plugin": ["ome_zarr = ome_zarr.napari"], }, extras_require={"napari": ["napari"]}, tests_require=["pytest", "pytest-capturelog"], diff --git a/tests/test_ome_zarr.py b/tests/test_ome_zarr.py index c56942b3..c9dd5cc5 100644 --- a/tests/test_ome_zarr.py +++ b/tests/test_ome_zarr.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -from ome_zarr import napari_get_reader, info, download +from ome_zarr.napari import napari_get_reader +from ome_zarr.utils import info, download from .create_test_data import create_zarr import tempfile import os From aafd5edc71145b44fc630b4353d9332af0889297 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 19 Aug 2020 12:29:09 +0200 Subject: [PATCH 03/39] Fix mypy errors --- .pre-commit-config.yaml | 1 - ome_zarr/reader.py | 40 ++++++++++++++++++++-------------------- setup.py | 3 ++- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 479f2642..9b26afd4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,6 @@ repos: rev: v0.782 hooks: - id: mypy - files: ome_zarr.py - repo: https://github.com/adrienverge/yamllint.git rev: v1.24.2 hooks: diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 367c6599..594efd5c 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -101,29 +101,29 @@ def reader_function( elif self.is_ome_label(): LOGGER.debug(f"treating {path} as labels") layers = self.load_ome_labels() - rv = [] - try: - for layer in layers: - metadata = layer[1].get("metadata", {}) - path = metadata.get("path", None) - array = metadata.get("image", {}).get("array", None) - if recurse and path and array: - # This is an ome mask, load the image - parent = posixpath.normpath(f"{path}/{array}") - LOGGER.debug(f"delegating to parent image: {parent}") - # Create a new OME Zarr Reader to load labels - replace = self.__class__(parent).reader_function( - None, recurse=False - ) + rv: List[LayerData] = [] + + for layer in layers: + metadata = layer[1].get("metadata", {}) + path = metadata.get("path", None) + array = metadata.get("image", {}).get("array", None) + if recurse and path and array: + # This is an ome mask, load the image + parent = posixpath.normpath(f"{path}/{array}") + LOGGER.debug(f"delegating to parent image: {parent}") + # Create a new OME Zarr Reader to load labels + replace = self.__class__(parent).reader_function( + None, recurse=False + ) + if replace: for r in replace: - r[1]["visible"] = False + if len(r) > 1: + r = cast(Tuple[Any, Dict], r) + r[1]["visible"] = False rv.extend(replace) layer[1]["visible"] = True - rv.append(layer) - return rv - except Exception as e: - LOGGER.error(e) - return [] + rv.append(layer) + return rv # TODO: might also be an individiaul mask diff --git a/setup.py b/setup.py index 282d8fd6..f1a98f83 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import os import codecs from setuptools import setup +from typing import List def read(fname): @@ -11,7 +12,7 @@ def read(fname): return codecs.open(file_path, encoding="utf-8").read() -install_requires = [] +install_requires: List[List[str]] = [] install_requires += (["numpy"],) install_requires += (["dask"],) install_requires += (["zarr"],) From 185157fb04a1f3cdef6968ad32c4edfacf1a66da Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 19 Aug 2020 13:36:58 +0200 Subject: [PATCH 04/39] Refactor to re-use multiscale loading --- ome_zarr/reader.py | 120 ++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 594efd5c..47e091fb 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -17,6 +17,7 @@ LOGGER = logging.getLogger("ome_zarr.reader") +# START type hint stuff LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] PathLike = Union[str, List[str]] # END type hint stuff. @@ -55,7 +56,8 @@ def has_ome_labels(self): "Does the zarr Image also include /labels sub-dir" return self.get_json("labels/.zgroup") - def is_ome_label(self): + def is_ome_labels_group(self): + # TODO: also check for "labels" entry and perhaps version? return self.zarr_path.endswith("labels/") and self.get_json(".zgroup") def get_label_names(self): @@ -77,6 +79,22 @@ def to_rgba(self, v): """Get rgba (0-1) e.g. (1, 0.5, 0, 1) from integer""" return [x / 255 for x in v.to_bytes(4, signed=True, byteorder="big")] + def update_metadata(self, data: LayerData, **kwargs): + """Cast LayerData for setting metadata""" + # Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] + if not data: + return () + elif len(data) == 1: # Tuple[Any] + return (data[0], dict(kwargs)) + else: + data = cast(Tuple[Any, Dict], data) + data[1].update(kwargs) + return data + + def new_reader(self, path: str, recurse: bool = False) -> Optional[List[LayerData]]: + """Create a new reader for the given path""" + return self.__class__(path).reader_function(None, recurse=recurse) + def reader_function( self, path: Optional[PathLike], recurse: bool = True, ) -> Optional[List[LayerData]]: @@ -91,38 +109,44 @@ def reader_function( layers = [self.load_ome_zarr()] # If the Image contains labels... if recurse and self.has_ome_labels(): - label_path = os.path.join(self.zarr_path, "labels") + labels_path = os.path.join(self.zarr_path, "labels") # Create a new OME Zarr Reader to load labels - labels = self.__class__(label_path).reader_function(None, recurse=False) + labels = self.new_reader(labels_path) if labels: layers.extend(labels) return layers - elif self.is_ome_label(): + elif self.is_ome_labels_group(): + LOGGER.debug(f"treating {path} as labels") - layers = self.load_ome_labels() + label_names = self.get_label_names() rv: List[LayerData] = [] + for name in label_names: + + # Load multiscale as well as label metadata + label_path: str = os.path.join(self.zarr_path, name) + multiscales: Optional[List[LayerData]] = self.new_reader(label_path) + if not multiscales: + continue + metadata: Dict = self.load_ome_label_metadata(name) - for layer in layers: - metadata = layer[1].get("metadata", {}) + # Look parent path = metadata.get("path", None) - array = metadata.get("image", {}).get("array", None) - if recurse and path and array: + image = metadata.get("image", {}).get("array", None) + if recurse and path and image: # This is an ome mask, load the image - parent = posixpath.normpath(f"{path}/{array}") + parent = posixpath.normpath(f"{path}/{image}") LOGGER.debug(f"delegating to parent image: {parent}") # Create a new OME Zarr Reader to load labels - replace = self.__class__(parent).reader_function( - None, recurse=False - ) + replace = self.new_reader(parent) if replace: + # Set replacements to be invisible for r in replace: - if len(r) > 1: - r = cast(Tuple[Any, Dict], r) - r[1]["visible"] = False + r = self.update_metadata(r, visible=False) rv.extend(replace) - layer[1]["visible"] = True - rv.append(layer) + for multiscale in multiscales: + multiscale = self.update_metadata(multiscale, visible=True) + rv.extend(multiscales) return rv # TODO: might also be an individiaul mask @@ -240,45 +264,27 @@ def load_ome_zarr(self): metadata = self.load_omero_metadata(data.shape[1]) return (pyramid, {"channel_axis": 1, **metadata}) - def load_ome_labels(self): - # look for labels in this dir... - label_names = self.get_label_names() - labels = [] - for name in label_names: - label_path = os.path.join(self.zarr_path, name) - label_attrs = self.get_json(f"{name}/.zattrs") - colors = {} - if "color" in label_attrs: - color_dict = label_attrs.get("color") - colors = dict() - for k, v in color_dict.items(): - try: - if k in ("true", "false"): - k = bool(k) - else: - k = int(k) - colors[k] = self.to_rgba(v) - except Exception as e: - LOGGER.error(f"invalid color - {k}={v}: {e}") - data = da.from_zarr(label_path) - # Split labels into separate channels, 1 per layer - for n in range(data.shape[1]): - labels.append( - ( - data[:, n, :, :, :], - { - "visible": False, - "name": name, - "color": colors, - "metadata": { - "image": label_attrs.get("image", {}), - "path": label_path, - }, - }, - "labels", - ) - ) - return labels + def load_ome_label_metadata(self, name: str): + # Metadata: TODO move to a class + label_attrs = self.get_json(f"{name}/.zattrs") + colors: Dict[Union[int, bool], str] = {} + if "color" in label_attrs: + color_dict = label_attrs.get("color") + for k, v in color_dict.items(): + try: + if k in ("true", "false"): + k = bool(k) + else: + k = int(k) + colors[k] = self.to_rgba(v) + except Exception as e: + LOGGER.error(f"invalid color - {k}={v}: {e}") + return { + "visible": False, + "name": name, + "color": colors, + "metadata": {"image": label_attrs.get("image", {}), "path": name}, + } class LocalZarr(BaseZarr): From 266b3573aa7e26a78162d3671e2bc8ff12c5966b Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 19 Aug 2020 14:53:23 +0200 Subject: [PATCH 05/39] Have mypy check all API methods --- .pre-commit-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b26afd4..55f45d7e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,15 @@ repos: rev: v0.782 hooks: - id: mypy + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.782 + hooks: + - id: mypy + args: [ + --disallow-untyped-defs, + --ignore-missing-imports, + ] + exclude: tests/*|setup.py - repo: https://github.com/adrienverge/yamllint.git rev: v1.24.2 hooks: From fc8df8f9408fcbef74d006edd1cced57fe170a3d Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 19 Aug 2020 15:23:06 +0200 Subject: [PATCH 06/39] Enfore mypy annotations on all definitions --- .pre-commit-config.yaml | 4 ++-- ome_zarr/cli.py | 8 +++---- ome_zarr/napari.py | 23 ++++++++----------- ome_zarr/reader.py | 51 +++++++++++++++++++++-------------------- ome_zarr/utils.py | 7 +++--- 5 files changed, 46 insertions(+), 47 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55f45d7e..4d7f943e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,8 +24,8 @@ repos: hooks: - id: mypy args: [ - --disallow-untyped-defs, - --ignore-missing-imports, + --disallow-untyped-defs, + --ignore-missing-imports, ] exclude: tests/*|setup.py - repo: https://github.com/adrienverge/yamllint.git diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index d47deeab..10b4f823 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -7,24 +7,24 @@ from .utils import download as zarr_download -def config_logging(loglevel, args): +def config_logging(loglevel: int, args: argparse.Namespace) -> None: loglevel = loglevel - (10 * args.verbose) + (10 * args.quiet) logging.basicConfig(level=loglevel) # DEBUG logging for s3fs so we can track remote calls logging.getLogger("s3fs").setLevel(logging.DEBUG) -def info(args): +def info(args: argparse.Namespace) -> None: config_logging(logging.INFO, args) zarr_info(args.path) -def download(args): +def download(args: argparse.Namespace) -> None: config_logging(logging.WARN, args) zarr_download(args.path, args.output, args.name) -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( "-v", diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 06285ffb..30f5f6c0 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -6,28 +6,25 @@ """ +import logging + +from typing import Any, Callable, Optional +from .reader import PathLike, ReaderFunction +from .utils import parse_url + + try: from napari_plugin_engine import napari_hook_implementation except ImportError: - def napari_hook_implementation(func, *args, **kwargs): + def napari_hook_implementation( + func: Callable, *args: Any, **kwargs: Any + ) -> Callable: return func -import logging - -# for optional type hints only, otherwise you can delete/ignore this stuff -from typing import List, Optional, Union, Any, Tuple, Dict, Callable - -from .utils import parse_url - LOGGER = logging.getLogger("ome_zarr.napari") -LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] -PathLike = Union[str, List[str]] -ReaderFunction = Callable[[PathLike], List[LayerData]] -# END type hint stuff. - @napari_hook_implementation def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 47e091fb..e9498298 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -13,18 +13,19 @@ import logging # for optional type hints only, otherwise you can delete/ignore this stuff -from typing import List, Optional, Union, Any, Tuple, Dict, cast +from typing import List, Optional, Union, Any, Tuple, Dict, Callable, cast LOGGER = logging.getLogger("ome_zarr.reader") # START type hint stuff LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] PathLike = Union[str, List[str]] +ReaderFunction = Callable[[PathLike], List[LayerData]] # END type hint stuff. class BaseZarr: - def __init__(self, path): + def __init__(self, path: str) -> None: self.zarr_path = path.endswith("/") and path or f"{path}/" self.zarray = self.get_json(".zarray") self.zgroup = self.get_json(".zgroup") @@ -38,7 +39,7 @@ def __init__(self, path): warnings.warn("deprecated loading of omero.josn", DeprecationWarning) self.image_data = self.get_json("omero.json") - def __str__(self): + def __str__(self) -> str: suffix = "" if self.zgroup: suffix += " [zgroup]" @@ -46,44 +47,44 @@ def __str__(self): suffix += " [zarray]" return f"{self.zarr_path}{suffix}" - def is_zarr(self): + def is_zarr(self) -> Optional[Dict]: return self.zarray or self.zgroup - def is_ome_zarr(self): - return self.zgroup and "multiscales" in self.root_attrs + def is_ome_zarr(self) -> bool: + return bool(self.zgroup) and "multiscales" in self.root_attrs - def has_ome_labels(self): + def has_ome_labels(self) -> Dict: "Does the zarr Image also include /labels sub-dir" return self.get_json("labels/.zgroup") - def is_ome_labels_group(self): + def is_ome_labels_group(self) -> bool: # TODO: also check for "labels" entry and perhaps version? - return self.zarr_path.endswith("labels/") and self.get_json(".zgroup") + return self.zarr_path.endswith("labels/") and bool(self.get_json(".zgroup")) - def get_label_names(self): + def get_label_names(self) -> List[str]: """ Called if is_ome_label is true """ # If this is a label, the names are in root .zattrs return self.root_attrs.get("labels", []) - def get_json(self, subpath): + def get_json(self, subpath: str) -> Dict: raise NotImplementedError("unknown") - def get_reader_function(self): + def get_reader_function(self) -> Callable: if not self.is_zarr(): raise Exception(f"not a zarr: {self}") return self.reader_function - def to_rgba(self, v): + def to_rgba(self, v: int) -> List[float]: """Get rgba (0-1) e.g. (1, 0.5, 0, 1) from integer""" return [x / 255 for x in v.to_bytes(4, signed=True, byteorder="big")] - def update_metadata(self, data: LayerData, **kwargs): + def update_metadata(self, data: LayerData, **kwargs: Any) -> LayerData: """Cast LayerData for setting metadata""" # Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] if not data: - return () + return None elif len(data) == 1: # Tuple[Any] return (data[0], dict(kwargs)) else: @@ -160,9 +161,9 @@ def reader_function( LOGGER.debug(f"ignoring {path}") return None - def load_omero_metadata(self, assert_channel_count=None): + def load_omero_metadata(self, assert_channel_count: int = None) -> Dict: """Load OMERO metadata as json and convert for napari""" - metadata = {} + metadata: Dict[str, Any] = {} try: model = "unknown" rdefs = self.image_data.get("rdefs", {}) @@ -190,7 +191,7 @@ def load_omero_metadata(self, assert_channel_count=None): return {} colormaps = [] - contrast_limits = [None for x in channels] + contrast_limits: Optional[List[Optional[Any]]] = [None for x in channels] names = [("channel_%d" % idx) for idx, ch in enumerate(channels)] visibles = [True for x in channels] @@ -231,7 +232,7 @@ def load_omero_metadata(self, assert_channel_count=None): return metadata - def load_ome_zarr(self): + def load_ome_zarr(self) -> LayerData: resolutions = ["0"] # TODO: could be first alphanumeric dataset on err try: @@ -264,12 +265,12 @@ def load_ome_zarr(self): metadata = self.load_omero_metadata(data.shape[1]) return (pyramid, {"channel_axis": 1, **metadata}) - def load_ome_label_metadata(self, name: str): + def load_ome_label_metadata(self, name: str) -> Dict: # Metadata: TODO move to a class label_attrs = self.get_json(f"{name}/.zattrs") - colors: Dict[Union[int, bool], str] = {} - if "color" in label_attrs: - color_dict = label_attrs.get("color") + colors: Dict[Union[int, bool], List[float]] = {} + color_dict = label_attrs.get("color", {}) + if color_dict: for k, v in color_dict.items(): try: if k in ("true", "false"): @@ -288,7 +289,7 @@ def load_ome_label_metadata(self, name: str): class LocalZarr(BaseZarr): - def get_json(self, subpath): + def get_json(self, subpath: str) -> Dict: filename = os.path.join(self.zarr_path, subpath) if not os.path.exists(filename): @@ -299,7 +300,7 @@ def get_json(self, subpath): class RemoteZarr(BaseZarr): - def get_json(self, subpath): + def get_json(self, subpath: str) -> Dict: url = f"{self.zarr_path}{subpath}" try: rsp = requests.get(url) diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py index c0583cec..9c7b6add 100644 --- a/ome_zarr/utils.py +++ b/ome_zarr/utils.py @@ -13,6 +13,7 @@ import logging from .reader import ( + BaseZarr, LocalZarr, RemoteZarr, ) @@ -21,7 +22,7 @@ LOGGER = logging.getLogger("ome_zarr.utils") -def parse_url(path): +def parse_url(path: str) -> BaseZarr: # Check is path is local directory first if os.path.isdir(path): return LocalZarr(path) @@ -34,7 +35,7 @@ def parse_url(path): return RemoteZarr(path) -def info(path): +def info(path: str) -> None: """ print information about the ome-zarr fileset """ @@ -47,7 +48,7 @@ def info(path): LOGGER.debug(data) -def download(path, output_dir=".", zarr_name=""): +def download(path: str, output_dir: str = ".", zarr_name: str = "") -> None: """ download zarr from URL """ From db43af52efb5b29d0201707adcd7329214c50717 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 19 Aug 2020 17:45:22 +0200 Subject: [PATCH 07/39] Fix 'bool(false)' in color test --- ome_zarr/reader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index e9498298..488b8cee 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -273,8 +273,10 @@ def load_ome_label_metadata(self, name: str) -> Dict: if color_dict: for k, v in color_dict.items(): try: - if k in ("true", "false"): - k = bool(k) + if k.lower() == "true": + k = True + elif k.lower() == "false": + k = False else: k = int(k) colors[k] = self.to_rgba(v) From 967b36cdd3c846c7fa94ef18372c400cb62a9a37 Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 21 Aug 2020 14:59:00 +0200 Subject: [PATCH 08/39] move create_test_data to ome_zarr.data --- tests/create_test_data.py => ome_zarr/data.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/create_test_data.py => ome_zarr/data.py (100%) diff --git a/tests/create_test_data.py b/ome_zarr/data.py similarity index 100% rename from tests/create_test_data.py rename to ome_zarr/data.py From c1461014bc52c2e6533c196bc13701bbfcfb3361 Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 21 Aug 2020 15:05:33 +0200 Subject: [PATCH 09/39] Add isort as the first pre-commit step --- .isort.cfg | 2 ++ .pre-commit-config.yaml | 15 +++++++++++++++ ome_zarr/cli.py | 2 +- ome_zarr/data.py | 2 +- ome_zarr/napari.py | 3 +-- ome_zarr/reader.py | 15 +++++++-------- ome_zarr/utils.py | 17 +++++------------ setup.py | 5 +++-- tests/test_ome_zarr.py | 10 ++++++---- 9 files changed, 41 insertions(+), 30 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..f9067812 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +known_third_party = dask,numpy,requests,setuptools,skimage,vispy,zarr diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d7f943e..158770aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,22 @@ --- repos: + + - repo: https://github.com/asottile/seed-isort-config + rev: v1.9.3 + hooks: + - id: seed-isort-config + + - repo: https://github.com/timothycrosley/isort + rev: 5.3.2 + hooks: + - id: isort + - repo: https://github.com/ambv/black rev: 19.10b0 hooks: - id: black args: [--target-version=py36] + - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: @@ -15,10 +27,12 @@ repos: # Conflicts with black: E203 whitespace before ':' --ignore=E203, ] + - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.782 hooks: - id: mypy + - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.782 hooks: @@ -28,6 +42,7 @@ repos: --ignore-missing-imports, ] exclude: tests/*|setup.py + - repo: https://github.com/adrienverge/yamllint.git rev: v1.24.2 hooks: diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index 10b4f823..caf7ed8a 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -3,8 +3,8 @@ import argparse import logging -from .utils import info as zarr_info from .utils import download as zarr_download +from .utils import info as zarr_info def config_logging(loglevel: int, args: argparse.Namespace) -> None: diff --git a/ome_zarr/data.py b/ome_zarr/data.py index 6378d939..d0f88c0c 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import zarr import numpy as np +import zarr from skimage import data from skimage.transform import pyramid_gaussian diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 30f5f6c0..21fb12c9 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -7,12 +7,11 @@ import logging - from typing import Any, Callable, Optional + from .reader import PathLike, ReaderFunction from .utils import parse_url - try: from napari_plugin_engine import napari_hook_implementation except ImportError: diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 488b8cee..06c58b06 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -1,19 +1,18 @@ """ Reading logic for ome-zarr """ -import os import json -import requests -import dask.array as da +import logging +import os import posixpath import warnings -from vispy.color import Colormap - -import logging - # for optional type hints only, otherwise you can delete/ignore this stuff -from typing import List, Optional, Union, Any, Tuple, Dict, Callable, cast +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast + +import dask.array as da +import requests +from vispy.color import Colormap LOGGER = logging.getLogger("ome_zarr.reader") diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py index 9c7b6add..f84dcb27 100644 --- a/ome_zarr/utils.py +++ b/ome_zarr/utils.py @@ -2,22 +2,15 @@ Utility methods for ome_zarr access """ -import os import json -import dask.array as da - -from dask.diagnostics import ProgressBar - -from urllib.parse import urlparse - import logging +import os +from urllib.parse import urlparse -from .reader import ( - BaseZarr, - LocalZarr, - RemoteZarr, -) +import dask.array as da +from dask.diagnostics import ProgressBar +from .reader import BaseZarr, LocalZarr, RemoteZarr LOGGER = logging.getLogger("ome_zarr.utils") diff --git a/setup.py b/setup.py index f1a98f83..8ee34298 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os import codecs -from setuptools import setup +import os from typing import List +from setuptools import setup + def read(fname): file_path = os.path.join(os.path.dirname(__file__), fname) diff --git a/tests/test_ome_zarr.py b/tests/test_ome_zarr.py index c9dd5cc5..e73daff7 100644 --- a/tests/test_ome_zarr.py +++ b/tests/test_ome_zarr.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- +import logging +import os +import tempfile + from ome_zarr.napari import napari_get_reader -from ome_zarr.utils import info, download +from ome_zarr.utils import download, info + from .create_test_data import create_zarr -import tempfile -import os -import logging def log_strings(idx, t, c, z, y, x, ct, cc, cz, cy, cx, dtype): From 1c50a26ed710f300f3b028c0203bf45d6ee381f8 Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 21 Aug 2020 16:05:00 +0200 Subject: [PATCH 10/39] refactor create_zarr method for use from CLI --- .isort.cfg | 2 +- ome_zarr/cli.py | 13 ++++++- ome_zarr/data.py | 91 +++++++++++++++++++++++++++++++++++++----------- 3 files changed, 84 insertions(+), 22 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index f9067812..626bc53d 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -known_third_party = dask,numpy,requests,setuptools,skimage,vispy,zarr +known_third_party = dask,numpy,requests,scipy,setuptools,skimage,vispy,zarr diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index caf7ed8a..f057a64e 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -3,6 +3,7 @@ import argparse import logging +from .data import create_zarr as zarr_coins from .utils import download as zarr_download from .utils import info as zarr_info @@ -24,6 +25,11 @@ def download(args: argparse.Namespace) -> None: zarr_download(args.path, args.output, args.name) +def coins(args: argparse.Namespace) -> None: + config_logging(logging.INFO, args) + zarr_coins(args.path) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( @@ -43,7 +49,7 @@ def main() -> None: subparsers = parser.add_subparsers(dest="command") subparsers.required = True - # foo + # info parser_info = subparsers.add_parser("info") parser_info.add_argument("path") parser_info.set_defaults(func=info) @@ -55,5 +61,10 @@ def main() -> None: parser_download.add_argument("--name", default="") parser_download.set_defaults(func=download) + # coin + parser_coins = subparsers.add_parser("coins") + parser_coins.add_argument("path") + parser_coins.set_defaults(func=coins) + args = parser.parse_args() args.func(args) diff --git a/ome_zarr/data.py b/ome_zarr/data.py index d0f88c0c..ef92ad77 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -1,33 +1,72 @@ #!/usr/bin/env python +from typing import List, Tuple + import numpy as np import zarr +from scipy.ndimage import zoom from skimage import data -from skimage.transform import pyramid_gaussian +from skimage.filters import threshold_otsu +from skimage.measure import label +from skimage.morphology import closing, remove_small_objects, square +from skimage.segmentation import clear_border -def create_zarr(zarr_directory): +def coins() -> Tuple[List, List]: + """ + Sample data from skimage + """ + # Thanks to Juan + # https://gist.github.com/jni/62e07ddd135dbb107278bc04c0f9a8e7 + image = data.coins()[50:-50, 50:-50] + thresh = threshold_otsu(image) + bw = closing(image > thresh, square(4)) + cleared = remove_small_objects(clear_border(bw), 20) + label_image = label(cleared) + pyramid = list(reversed([zoom(image, 2 ** i, order=3) for i in range(4)])) + labels = list(reversed([zoom(label_image, 2 ** i, order=0) for i in range(4)])) + return pyramid, labels - base = np.tile(data.astronaut(), (2, 2, 1)) - gaussian = list(pyramid_gaussian(base, downscale=2, max_layer=4, multichannel=True)) - pyramid = [] - # convert each level of pyramid into 5D image (t, c, z, y, x) - for pixels in gaussian: - red = pixels[:, :, 0] - green = pixels[:, :, 1] - blue = pixels[:, :, 2] - # wrap to make 5D: (t, c, z, y, x) - pixels = np.array([np.array([red]), np.array([green]), np.array([blue])]) - pixels = np.array([pixels]) - pyramid.append(pixels) +def rgba_to_int(r: int, g: int, b: int, a: int) -> int: + return int.from_bytes([r, g, b, a], byteorder="big", signed=True) + + +def rgb_to_5d(pixels: np.ndarray) -> List: + """convert an RGB image into 5D image (t, c, z, y, x)""" + if len(pixels.shape) == 2: + channels = [[np.array(pixels)]] + elif len(pixels.shape) == 3: + size_c = pixels.shape(2) + channels = [np.array(pixels[:, :, c]) for c in range(size_c)] + else: + assert f"expecting 2 or 3d: ({pixels.shape})" + return [np.array(channels)] + + +def write_multiscale(pyramid: List, group: zarr.Group) -> None: - store = zarr.DirectoryStore(zarr_directory) - grp = zarr.group(store) paths = [] for path, dataset in enumerate(pyramid): - grp.create_dataset(str(path), data=pyramid[path]) + group.create_dataset(str(path), data=pyramid[path]) paths.append({"path": str(path)}) + multiscales = [{"version": "0.1", "datasets": paths}] + group.attrs["multiscales"] = multiscales + + +def create_zarr(zarr_directory: str) -> None: + + pyramid, labels = coins() + pyramid = [rgb_to_5d(layer) for layer in pyramid] + labels = [rgb_to_5d(layer) for layer in labels] + + store = zarr.DirectoryStore(zarr_directory) + grp = zarr.group(store) + write_multiscale(pyramid, grp) + + labels_grp = grp.create_group("labels") + labels_grp.attrs["labels"] = ["coins"] + image_data = { "id": 1, "channels": [ @@ -52,7 +91,19 @@ def create_zarr(zarr_directory): ], "rdefs": {"model": "color"}, } + if False: # FIXME + grp.attrs["omero"] = image_data - multiscales = [{"version": "0.1", "datasets": paths}] - grp.attrs["multiscales"] = multiscales - grp.attrs["omero"] = image_data + coins_grp = labels_grp.create_group("coins") + write_multiscale(labels, coins_grp) + coins_grp.attrs["color"] = { + "1": rgba_to_int(50, 0, 0, 0), + "2": rgba_to_int(0, 50, 0, 0), + "3": rgba_to_int(0, 0, 50, 0), + "4": rgba_to_int(100, 0, 0, 0), + "5": rgba_to_int(0, 100, 0, 0), + "6": rgba_to_int(0, 0, 100, 0), + "7": rgba_to_int(50, 50, 50, 0), + "8": rgba_to_int(100, 100, 100, 0), + } + coins_grp.attrs["image"] = {"array": "../../", "source": {}} From 3a9f360b18ed1c32715339324b3e8980fd82a1b5 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 24 Aug 2020 22:57:05 +0200 Subject: [PATCH 11/39] Major API refactoring The introduction of multiple types per node of the Zarr hierarchy (e.g. multiscales AND labeled image) made the existing code flow difficult to extend and test. This reworks the previous BaseZarr type into multiple hierarches: - ome_zarr.io: Locations hierarchy for choosing Remote or Local - ome_zarr.reader: Spec hierarchy for choosing metadata types The general code flow consists of: - ome_zarr.napari.napari_get_reader() - ome_zarr.io.parse_url() - ome_zarr.reader.Reader.__call__() - ome_zarr.reader.Layer() -ome_zarr.reader.Spec() [recursion] - ome_zarr.napari.transform --- ome_zarr/cli.py | 12 +- ome_zarr/conversions.py | 10 + ome_zarr/data.py | 39 ++- ome_zarr/io.py | 108 ++++++++ ome_zarr/napari.py | 30 ++- ome_zarr/reader.py | 476 +++++++++++++++++----------------- ome_zarr/types.py | 13 + ome_zarr/utils.py | 70 +++-- tests/test_cli.py | 17 ++ tests/test_layer.py | 32 +++ tests/test_napari.py | 42 +++ tests/test_ome_zarr.py | 11 +- tests/test_reader.py | 36 +++ tests/test_starting_points.py | 45 ++++ 14 files changed, 637 insertions(+), 304 deletions(-) create mode 100644 ome_zarr/conversions.py create mode 100644 ome_zarr/io.py create mode 100644 ome_zarr/types.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_layer.py create mode 100644 tests/test_napari.py create mode 100644 tests/test_reader.py create mode 100644 tests/test_starting_points.py diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index f057a64e..e0d34533 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -2,6 +2,8 @@ import argparse import logging +import sys +from typing import List from .data import create_zarr as zarr_coins from .utils import download as zarr_download @@ -30,7 +32,8 @@ def coins(args: argparse.Namespace) -> None: zarr_coins(args.path) -def main() -> None: +def main(args: List[str] = None) -> None: + parser = argparse.ArgumentParser() parser.add_argument( "-v", @@ -66,5 +69,8 @@ def main() -> None: parser_coins.add_argument("path") parser_coins.set_defaults(func=coins) - args = parser.parse_args() - args.func(args) + if args is None: + ns = parser.parse_args(sys.argv) + else: + ns = parser.parse_args(args) + ns.func(ns) diff --git a/ome_zarr/conversions.py b/ome_zarr/conversions.py new file mode 100644 index 00000000..a13ff3a1 --- /dev/null +++ b/ome_zarr/conversions.py @@ -0,0 +1,10 @@ +from typing import List + + +def int_to_rgba(v: int) -> List[float]: + """Get rgba (0-1) e.g. (1, 0.5, 0, 1) from integer""" + return [x / 255 for x in v.to_bytes(4, signed=True, byteorder="big")] + + +def rgba_to_int(r: int, g: int, b: int, a: int) -> int: + return int.from_bytes([r, g, b, a], byteorder="big", signed=True) diff --git a/ome_zarr/data.py b/ome_zarr/data.py index ef92ad77..ac841ba4 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from typing import List, Tuple +from typing import Callable, List, Tuple import numpy as np import zarr @@ -9,6 +9,9 @@ from skimage.measure import label from skimage.morphology import closing, remove_small_objects, square from skimage.segmentation import clear_border +from skimage.transform import pyramid_gaussian + +from .conversions import rgba_to_int def coins() -> Tuple[List, List]: @@ -22,13 +25,30 @@ def coins() -> Tuple[List, List]: bw = closing(image > thresh, square(4)) cleared = remove_small_objects(clear_border(bw), 20) label_image = label(cleared) + pyramid = list(reversed([zoom(image, 2 ** i, order=3) for i in range(4)])) labels = list(reversed([zoom(label_image, 2 ** i, order=0) for i in range(4)])) + + pyramid = [rgb_to_5d(layer) for layer in pyramid] + labels = [rgb_to_5d(layer) for layer in labels] return pyramid, labels -def rgba_to_int(r: int, g: int, b: int, a: int) -> int: - return int.from_bytes([r, g, b, a], byteorder="big", signed=True) +def astronaut() -> Tuple[List, List]: + base = np.tile(data.astronaut(), (2, 2, 1)) + gaussian = list(pyramid_gaussian(base, downscale=2, max_layer=4, multichannel=True)) + + pyramid = [] + # convert each level of pyramid into 5D image (t, c, z, y, x) + for pixels in gaussian: + red = pixels[:, :, 0] + green = pixels[:, :, 1] + blue = pixels[:, :, 2] + # wrap to make 5D: (t, c, z, y, x) + pixels = np.array([np.array([red]), np.array([green]), np.array([blue])]) + pixels = np.array([pixels]) + pyramid.append(pixels) + return pyramid, [] def rgb_to_5d(pixels: np.ndarray) -> List: @@ -39,7 +59,7 @@ def rgb_to_5d(pixels: np.ndarray) -> List: size_c = pixels.shape(2) channels = [np.array(pixels[:, :, c]) for c in range(size_c)] else: - assert f"expecting 2 or 3d: ({pixels.shape})" + assert False, f"expecting 2 or 3d: ({pixels.shape})" return [np.array(channels)] @@ -54,11 +74,11 @@ def write_multiscale(pyramid: List, group: zarr.Group) -> None: group.attrs["multiscales"] = multiscales -def create_zarr(zarr_directory: str) -> None: +def create_zarr( + zarr_directory: str, method: Callable[..., Tuple[List, List]] = coins +) -> None: - pyramid, labels = coins() - pyramid = [rgb_to_5d(layer) for layer in pyramid] - labels = [rgb_to_5d(layer) for layer in labels] + pyramid, labels = method() store = zarr.DirectoryStore(zarr_directory) grp = zarr.group(store) @@ -91,8 +111,7 @@ def create_zarr(zarr_directory: str) -> None: ], "rdefs": {"model": "color"}, } - if False: # FIXME - grp.attrs["omero"] = image_data + grp.attrs["omero"] = image_data coins_grp = labels_grp.create_group("coins") write_multiscale(labels, coins_grp) diff --git a/ome_zarr/io.py b/ome_zarr/io.py new file mode 100644 index 00000000..ca6d76d7 --- /dev/null +++ b/ome_zarr/io.py @@ -0,0 +1,108 @@ +""" +Reading logic for ome-zarr +""" + +import json +import logging +import os +import posixpath +from abc import ABC, abstractmethod +from typing import Optional +from urllib.parse import urlparse + +import dask.array as da +import requests + +from .types import JSONDict + +LOGGER = logging.getLogger("ome_zarr.io") + + +class BaseZarrLocation(ABC): + def __init__(self, path: str) -> None: + self.zarr_path: str = path.endswith("/") and path or f"{path}/" + self.zarray: JSONDict = self.get_json(".zarray") + self.zgroup: JSONDict = self.get_json(".zgroup") + self.root_attrs: JSONDict = {} + if self.zgroup: + self.root_attrs = self.get_json(".zattrs") + elif self.zarray: + self.root_attrs = self.get_json(".zattrs") + + def __repr__(self) -> str: + suffix = "" + if self.zgroup: + suffix += " [zgroup]" + if self.zarray: + suffix += " [zarray]" + return f"{self.zarr_path}{suffix}" + + def exists(self) -> bool: + return os.path.exists(self.zarr_path) + + def is_zarr(self) -> Optional[JSONDict]: + return self.zarray or self.zgroup + + @abstractmethod + def get_json(self, subpath: str) -> JSONDict: + raise NotImplementedError("unknown") + + def load(self, subpath: str) -> da.core.Array: + """ + Use dask.array.from_zarr to load the subpath + """ + return da.from_zarr(f"{self.zarr_path}{subpath}") + + # TODO: update to from __future__ import annotations with 3.7+ + def open(self, path: str) -> "BaseZarrLocation": + """Create a new zarr for the given path""" + return self.__class__(posixpath.normpath(f"{self.zarr_path}/{path}")) + + +class LocalZarrLocation(BaseZarrLocation): + def get_json(self, subpath: str) -> JSONDict: + filename = os.path.join(self.zarr_path, subpath) + + if not os.path.exists(filename): + return {} + + with open(filename) as f: + return json.loads(f.read()) + + +class RemoteZarrLocation(BaseZarrLocation): + def get_json(self, subpath: str) -> JSONDict: + url = f"{self.zarr_path}{subpath}" + try: + rsp = requests.get(url) + except Exception: + LOGGER.warn(f"unreachable: {url} -- details logged at debug") + LOGGER.debug("exception details:", exc_info=True) + return {} + try: + if rsp.status_code in (403, 404): # file doesn't exist + return {} + return rsp.json() + except Exception: + LOGGER.error(f"({rsp.status_code}): {rsp.text}") + return {} + + +def parse_url(path: str) -> Optional[BaseZarrLocation]: + """ convert a path string or URL to a BaseZarrLocation instance + >>> parse_url('does-not-exist') + """ + # Check is path is local directory first + if os.path.isdir(path): + return LocalZarrLocation(path) + else: + result = urlparse(path) + zarr: Optional[BaseZarrLocation] = None + if result.scheme in ("", "file"): + # Strips 'file://' if necessary + zarr = LocalZarrLocation(result.path) + else: + zarr = RemoteZarrLocation(path) + if zarr.is_zarr(): + return zarr + return None diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 21fb12c9..4ff1b878 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -7,10 +7,12 @@ import logging -from typing import Any, Callable, Optional +import warnings +from typing import Any, Callable, Iterator, List, Optional -from .reader import PathLike, ReaderFunction -from .utils import parse_url +from .io import parse_url +from .reader import Layer, Reader +from .types import LayerData, PathLike, ReaderFunction try: from napari_plugin_engine import napari_hook_implementation @@ -33,9 +35,25 @@ def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: - URL of the form: https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/ID.zarr/ """ if isinstance(path, list): + if len(path) > 1: + warnings.warn("more than one path is not currently supported") path = path[0] - instance = parse_url(path) - if instance is not None and instance.is_zarr(): - return instance.get_reader_function() + zarr = parse_url(path) + if zarr: + reader = Reader(zarr) + return transform(reader()) # Ignoring this path return None + + +def transform(layers: Iterator[Layer]) -> Optional[ReaderFunction]: + def f(*args: Any, **kwargs: Any) -> List[LayerData]: + results: List[LayerData] = list() + + for layer in layers: + data = layer.data + metadata = layer.metadata + results.append((data, {"channel_axis": 1, **metadata})) + return results + + return f diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 06c58b06..fd759cc1 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -1,168 +1,219 @@ """ Reading logic for ome-zarr """ -import json + import logging -import os import posixpath -import warnings - -# for optional type hints only, otherwise you can delete/ignore this stuff -from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast +from abc import ABC +from typing import Any, Dict, Iterator, List, Optional, Union import dask.array as da -import requests from vispy.color import Colormap +from .conversions import int_to_rgba +from .io import BaseZarrLocation +from .types import JSONDict + LOGGER = logging.getLogger("ome_zarr.reader") -# START type hint stuff -LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] -PathLike = Union[str, List[str]] -ReaderFunction = Callable[[PathLike], List[LayerData]] -# END type hint stuff. - - -class BaseZarr: - def __init__(self, path: str) -> None: - self.zarr_path = path.endswith("/") and path or f"{path}/" - self.zarray = self.get_json(".zarray") - self.zgroup = self.get_json(".zgroup") - if self.zgroup: - self.root_attrs = self.get_json(".zattrs") - if "omero" in self.root_attrs: - self.image_data = self.root_attrs["omero"] - # TODO: start checking metadata version - else: - # Backup location that can be removed in the future. - warnings.warn("deprecated loading of omero.josn", DeprecationWarning) - self.image_data = self.get_json("omero.json") + +class Layer: + """ + Container for a representation of the binary data somewhere in + the data hierarchy. + """ + + def __init__(self, zarr: BaseZarrLocation): + self.zarr = zarr + self.visible = True + + # Likely to be updated by specs + self.metadata: JSONDict = dict() + self.data: List[da.core.Array] = list() + self.specs: List[Spec] = [] + self.pre_layers: List[Layer] = [] + self.post_layers: List[Layer] = [] + + # TODO: this should be some form of plugin infra over subclasses + if Labels.matches(zarr): + self.specs.append(Labels(self)) + if Label.matches(zarr): + self.specs.append(Label(self)) + if Multiscales.matches(zarr): + self.specs.append(Multiscales(self)) + if OMERO.matches(zarr): + self.specs.append(OMERO(self)) + + def write_metadata(self, metadata: JSONDict) -> None: + for spec in self.specs: + metadata.update(self.zarr.root_attrs) def __str__(self) -> str: suffix = "" - if self.zgroup: + if self.zarr.zgroup: suffix += " [zgroup]" - if self.zarray: + if self.zarr.zarray: suffix += " [zarray]" - return f"{self.zarr_path}{suffix}" + return f"{self.zarr.zarr_path}{suffix}" + + +class Spec(ABC): + """ + Base class for specifications that can be implemented by groups + or arrays within the hierarchy. Multiple subclasses may apply. + """ - def is_zarr(self) -> Optional[Dict]: - return self.zarray or self.zgroup + @staticmethod + def matches(zarr: BaseZarrLocation) -> bool: + raise NotImplementedError() - def is_ome_zarr(self) -> bool: - return bool(self.zgroup) and "multiscales" in self.root_attrs + def __init__(self, layer: Layer) -> None: + self.layer = layer + self.zarr = layer.zarr + LOGGER.debug(f"treating {self.zarr} as {self.__class__.__name__}") + for k, v in self.zarr.root_attrs.items(): + LOGGER.info("root_attr: %s", k) + LOGGER.debug(v) - def has_ome_labels(self) -> Dict: - "Does the zarr Image also include /labels sub-dir" - return self.get_json("labels/.zgroup") + def lookup(self, key: str, default: Any) -> Any: + return self.zarr.root_attrs.get(key, default) - def is_ome_labels_group(self) -> bool: + +class Labels(Spec): + """ + Relatively small specification for the well-known "labels" group + which only contains the name of subgroups which should be loaded + an labeled images. + """ + + @staticmethod + def matches(zarr: BaseZarrLocation) -> bool: + """Does the Zarr Image group also include a /labels sub-group?""" # TODO: also check for "labels" entry and perhaps version? - return self.zarr_path.endswith("labels/") and bool(self.get_json(".zgroup")) + return bool("labels" in zarr.root_attrs) + + def __init__(self, layer: Layer) -> None: + super().__init__(layer) + label_names = self.lookup("labels", []) + for name in label_names: + child_zarr = self.zarr.open(name) + child_layer = Layer(child_zarr) + layer.post_layers.append(child_layer) + + +class Label(Spec): + """ + An additional aspect to a multiscale image is that it can be a labeled + image, in which each discrete pixel value represents a separate object. + """ - def get_label_names(self) -> List[str]: + @staticmethod + def matches(zarr: BaseZarrLocation) -> bool: """ - Called if is_ome_label is true + If label-specific metadata is present, then return true. """ - # If this is a label, the names are in root .zattrs - return self.root_attrs.get("labels", []) - - def get_json(self, subpath: str) -> Dict: - raise NotImplementedError("unknown") - - def get_reader_function(self) -> Callable: - if not self.is_zarr(): - raise Exception(f"not a zarr: {self}") - return self.reader_function - - def to_rgba(self, v: int) -> List[float]: - """Get rgba (0-1) e.g. (1, 0.5, 0, 1) from integer""" - return [x / 255 for x in v.to_bytes(4, signed=True, byteorder="big")] - - def update_metadata(self, data: LayerData, **kwargs: Any) -> LayerData: - """Cast LayerData for setting metadata""" - # Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] - if not data: - return None - elif len(data) == 1: # Tuple[Any] - return (data[0], dict(kwargs)) - else: - data = cast(Tuple[Any, Dict], data) - data[1].update(kwargs) - return data - - def new_reader(self, path: str, recurse: bool = False) -> Optional[List[LayerData]]: - """Create a new reader for the given path""" - return self.__class__(path).reader_function(None, recurse=recurse) - - def reader_function( - self, path: Optional[PathLike], recurse: bool = True, - ) -> Optional[List[LayerData]]: - """Take a path or list of paths and return a list of LayerData tuples.""" - - if isinstance(path, list): - path = path[0] - # TODO: safe to ignore this path? - - if self.is_ome_zarr(): - LOGGER.debug(f"treating {path} as ome-zarr") - layers = [self.load_ome_zarr()] - # If the Image contains labels... - if recurse and self.has_ome_labels(): - labels_path = os.path.join(self.zarr_path, "labels") - # Create a new OME Zarr Reader to load labels - labels = self.new_reader(labels_path) - if labels: - layers.extend(labels) - return layers - - elif self.is_ome_labels_group(): - - LOGGER.debug(f"treating {path} as labels") - label_names = self.get_label_names() - rv: List[LayerData] = [] - for name in label_names: - - # Load multiscale as well as label metadata - label_path: str = os.path.join(self.zarr_path, name) - multiscales: Optional[List[LayerData]] = self.new_reader(label_path) - if not multiscales: - continue - metadata: Dict = self.load_ome_label_metadata(name) - - # Look parent - path = metadata.get("path", None) - image = metadata.get("image", {}).get("array", None) - if recurse and path and image: - # This is an ome mask, load the image - parent = posixpath.normpath(f"{path}/{image}") - LOGGER.debug(f"delegating to parent image: {parent}") - # Create a new OME Zarr Reader to load labels - replace = self.new_reader(parent) - if replace: - # Set replacements to be invisible - for r in replace: - r = self.update_metadata(r, visible=False) - rv.extend(replace) - for multiscale in multiscales: - multiscale = self.update_metadata(multiscale, visible=True) - rv.extend(multiscales) - return rv - - # TODO: might also be an individiaul mask - - elif self.zarray: - LOGGER.debug(f"treating {path} as raw zarr") - data = da.from_zarr(f"{self.zarr_path}") - return [(data,)] + # FIXME: this should be the "label" metadata soon + return bool("colors" in zarr.root_attrs or "image" in zarr.root_attrs) + + def __init__(self, layer: Layer) -> None: + super().__init__(layer) + layer.visible = True + + path = self.lookup("path", None) + image = self.lookup("image", {}).get("array", None) + if path and image: + # This is an ome mask, load the image + parent = posixpath.normpath(f"{path}/{image}") + LOGGER.debug(f"delegating to parent image: {parent}") + parent_zarr = self.zarr.open(parent) + if parent_zarr.exists(): + parent_layer = Layer(parent_zarr) + layer.pre_layers.append(parent_layer) + layer.visible = False - else: - LOGGER.debug(f"ignoring {path}") - return None + # Metadata: TODO move to a class + colors: Dict[Union[int, bool], List[float]] = {} + color_dict = self.lookup("color", {}) + if color_dict: + for k, v in color_dict.items(): + try: + if k.lower() == "true": + k = True + elif k.lower() == "false": + k = False + else: + k = int(k) + colors[k] = int_to_rgba(v) + except Exception as e: + LOGGER.error(f"invalid color - {k}={v}: {e}") + + # TODO: a metadata transform should be provided by specific impls. + name = self.zarr.zarr_path.split("/")[-1] + layer.metadata.update( + { + "visible": False, + "name": name, + # "colormap": colors, + "metadata": {"image": self.lookup("image", {}), "path": name}, + } + ) + + +class Multiscales(Spec): + @staticmethod + def matches(zarr: BaseZarrLocation) -> bool: + """is multiscales metadata present?""" + if zarr.zgroup: + if "multiscales" in zarr.root_attrs: + return True + return False + + def __init__(self, layer: Layer) -> None: + super().__init__(layer) + + try: + datasets = self.lookup("multiscales", [])[0]["datasets"] + datasets = [d["path"] for d in datasets] + self.datasets: List[str] = datasets + LOGGER.info("datasets %s", datasets) + except Exception as e: + LOGGER.error(f"failed to parse multiscale metadata: {e}") + return # EARLY EXIT + + for resolution in self.datasets: + # data.shape is (t, c, z, y, x) by convention + data: da.core.Array = self.zarr.load(resolution) + chunk_sizes = [ + str(c[0]) + (" (+ %s)" % c[-1] if c[-1] != c[0] else "") + for c in data.chunks + ] + LOGGER.info("resolution: %s", resolution) + LOGGER.info(" - shape (t, c, z, y, x) = %s", data.shape) + LOGGER.info(" - chunks = %s", chunk_sizes) + LOGGER.info(" - dtype = %s", data.dtype) + layer.data.append(data) + + # TODO: test removal + if len(layer.data) == 1: + layer.data = layer.data[0] + + # Load possible layer data + child_zarr = self.zarr.open("labels") + # Creating a layer propagates to sub-specs, but the layer itself + # should not be registered. + Layer(child_zarr) + + +class OMERO(Spec): + @staticmethod + def matches(zarr: BaseZarrLocation) -> bool: + return bool("omero" in zarr.root_attrs) + + def __init__(self, layer: Layer) -> None: + super().__init__(layer) + # TODO: start checking metadata version + self.image_data = self.lookup("omero", {}) - def load_omero_metadata(self, assert_channel_count: int = None) -> Dict: - """Load OMERO metadata as json and convert for napari""" - metadata: Dict[str, Any] = {} try: model = "unknown" rdefs = self.image_data.get("rdefs", {}) @@ -171,23 +222,13 @@ def load_omero_metadata(self, assert_channel_count: int = None) -> Dict: channels = self.image_data.get("channels", None) if channels is None: - return {} + return # EARLY EXIT - count = None try: - count = len(channels) - if assert_channel_count: - if count != assert_channel_count: - LOGGER.error( - ( - "unexpected channel count: " - f"{count}!={assert_channel_count}" - ) - ) - return {} + len(channels) except Exception: LOGGER.warn(f"error counting channels: {channels}") - return {} + return # EARLY EXIT colormaps = [] contrast_limits: Optional[List[Optional[Any]]] = [None for x in channels] @@ -222,97 +263,50 @@ def load_omero_metadata(self, assert_channel_count: int = None) -> Dict: elif contrast_limits is not None: contrast_limits[idx] = [start, end] - metadata["colormap"] = colormaps - metadata["contrast_limits"] = contrast_limits - metadata["name"] = names - metadata["visible"] = visibles + layer.metadata["colormap"] = colormaps + layer.metadata["contrast_limits"] = contrast_limits + layer.metadata["name"] = names + layer.metadata["visible"] = visibles except Exception as e: LOGGER.error(f"failed to parse metadata: {e}") - return metadata - def load_ome_zarr(self) -> LayerData: +class Reader: + """ + Parses the given Zarr instance into a collection of Layers properly + ordered depending on context. Depending on the starting point, metadata + may be followed up or down the Zarr hierarchy. + """ + + def __init__(self, zarr: BaseZarrLocation) -> None: + assert zarr.is_zarr() + self.zarr = zarr + + def __call__(self) -> Iterator[Layer]: + layer = Layer(self.zarr) + if layer.specs: # Something has matched + LOGGER.debug(f"treating {self.zarr} as ome-zarr") + + # FIXME -- this will need recursion + for pre_layer in layer.pre_layers: + yield pre_layer + if layer.data: + yield layer + for post_layer in layer.post_layers: + yield post_layer + + # TODO: API thoughts for the Spec type + # - ask for earlier_layers, later_layers (i.e. priorities) + # - ask for recursion or not + # - ask for visible or invisible (?) + # - ask for "provides data", "overrides data" + + elif self.zarr.zarray: # Nothing has matched + LOGGER.debug(f"treating {self.zarr} as raw zarr") + data = da.from_zarr(f"{self.zarr.zarr_path}") + layer.data.append(data) + yield layer - resolutions = ["0"] # TODO: could be first alphanumeric dataset on err - try: - for k, v in self.root_attrs.items(): - LOGGER.info("root_attr: %s", k) - LOGGER.debug(v) - if "multiscales" in self.root_attrs: - datasets = self.root_attrs["multiscales"][0]["datasets"] - resolutions = [d["path"] for d in datasets] - except Exception as e: - raise e - - pyramid = [] - for resolution in resolutions: - # data.shape is (t, c, z, y, x) by convention - data = da.from_zarr(f"{self.zarr_path}{resolution}") - chunk_sizes = [ - str(c[0]) + (" (+ %s)" % c[-1] if c[-1] != c[0] else "") - for c in data.chunks - ] - LOGGER.info("resolution: %s", resolution) - LOGGER.info(" - shape (t, c, z, y, x) = %s", data.shape) - LOGGER.info(" - chunks = %s", chunk_sizes) - LOGGER.info(" - dtype = %s", data.dtype) - pyramid.append(data) - - if len(pyramid) == 1: - pyramid = pyramid[0] - - metadata = self.load_omero_metadata(data.shape[1]) - return (pyramid, {"channel_axis": 1, **metadata}) - - def load_ome_label_metadata(self, name: str) -> Dict: - # Metadata: TODO move to a class - label_attrs = self.get_json(f"{name}/.zattrs") - colors: Dict[Union[int, bool], List[float]] = {} - color_dict = label_attrs.get("color", {}) - if color_dict: - for k, v in color_dict.items(): - try: - if k.lower() == "true": - k = True - elif k.lower() == "false": - k = False - else: - k = int(k) - colors[k] = self.to_rgba(v) - except Exception as e: - LOGGER.error(f"invalid color - {k}={v}: {e}") - return { - "visible": False, - "name": name, - "color": colors, - "metadata": {"image": label_attrs.get("image", {}), "path": name}, - } - - -class LocalZarr(BaseZarr): - def get_json(self, subpath: str) -> Dict: - filename = os.path.join(self.zarr_path, subpath) - - if not os.path.exists(filename): - return {} - - with open(filename) as f: - return json.loads(f.read()) - - -class RemoteZarr(BaseZarr): - def get_json(self, subpath: str) -> Dict: - url = f"{self.zarr_path}{subpath}" - try: - rsp = requests.get(url) - except Exception: - LOGGER.warn(f"unreachable: {url} -- details logged at debug") - LOGGER.debug("exception details:", exc_info=True) - return {} - try: - if rsp.status_code in (403, 404): # file doesn't exist - return {} - return rsp.json() - except Exception: - LOGGER.error(f"({rsp.status_code}): {rsp.text}") - return {} + else: + LOGGER.debug(f"ignoring {self.zarr}") + # yield nothing diff --git a/ome_zarr/types.py b/ome_zarr/types.py new file mode 100644 index 00000000..078d77ae --- /dev/null +++ b/ome_zarr/types.py @@ -0,0 +1,13 @@ +""" +Definition of complex types for use elsewhere +""" + +from typing import Any, Callable, Dict, List, Tuple, Union + +LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] + +PathLike = Union[str, List[str]] + +ReaderFunction = Callable[[PathLike], List[LayerData]] + +JSONDict = Dict[str, Any] diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py index f84dcb27..421cba20 100644 --- a/ome_zarr/utils.py +++ b/ome_zarr/utils.py @@ -5,65 +5,58 @@ import json import logging import os -from urllib.parse import urlparse +from typing import List, Optional import dask.array as da from dask.diagnostics import ProgressBar -from .reader import BaseZarr, LocalZarr, RemoteZarr +from .io import parse_url +from .reader import OMERO, Layer, Multiscales +from .types import JSONDict LOGGER = logging.getLogger("ome_zarr.utils") -def parse_url(path: str) -> BaseZarr: - # Check is path is local directory first - if os.path.isdir(path): - return LocalZarr(path) - else: - result = urlparse(path) - if result.scheme in ("", "file"): - # Strips 'file://' if necessary - return LocalZarr(result.path) - else: - return RemoteZarr(path) - - -def info(path: str) -> None: +def info(path: str) -> Optional[Layer]: """ print information about the ome-zarr fileset """ zarr = parse_url(path) - if not zarr.is_ome_zarr(): - print(f"not an ome-zarr: {zarr}") - return - reader = zarr.get_reader_function() - data = reader(path) - LOGGER.debug(data) + if not zarr: + print(f"not a zarr: {zarr}") + return None + else: + layer = Layer(zarr) + if not layer.specs: + print(f"not an ome-zarr: {zarr}") + LOGGER.debug(layer.data) + return layer def download(path: str, output_dir: str = ".", zarr_name: str = "") -> None: """ download zarr from URL """ - omezarr = parse_url(path) - if not omezarr.is_ome_zarr(): - print(f"not an ome-zarr: {path}") + layer = info(path) + if not layer: return + image_id = "unknown" + resolutions: List[da.core.Array] = [] + datasets: List[str] = [] + for spec in layer.specs: + if isinstance(spec, OMERO): + image_id = spec.image_data.get("id", image_id) + if isinstance(spec, Multiscales): + datasets = spec.datasets + resolutions = layer.data + if not datasets or not resolutions: + print("no multiscales data found") + return - image_id = omezarr.image_data.get("id", "unknown") LOGGER.info("image_id %s", image_id) if not zarr_name: zarr_name = f"{image_id}.zarr" - try: - datasets = omezarr.root_attrs["multiscales"][0]["datasets"] - datasets = [x["path"] for x in datasets] - except KeyError: - datasets = ["0"] - LOGGER.info("datasets %s", datasets) - resolutions = [da.from_zarr(path, component=str(i)) for i in datasets] - # levels = list(range(len(resolutions))) - target_dir = os.path.join(output_dir, f"{zarr_name}") if os.path.exists(target_dir): print(f"{target_dir} already exists!") @@ -72,11 +65,14 @@ def download(path: str, output_dir: str = ".", zarr_name: str = "") -> None: pbar = ProgressBar() for dataset, data in reversed(list(zip(datasets, resolutions))): + print("X", layer, dataset, data) LOGGER.info(f"resolution {dataset}...") with pbar: data.to_zarr(os.path.join(target_dir, dataset)) with open(os.path.join(target_dir, ".zgroup"), "w") as f: - f.write(json.dumps(omezarr.zgroup)) + f.write(json.dumps(layer.zarr.zgroup)) with open(os.path.join(target_dir, ".zattrs"), "w") as f: - f.write(json.dumps(omezarr.root_attrs)) + metadata: JSONDict = {} + layer.write_metadata(metadata) + f.write(json.dumps(metadata)) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..b3a59cd3 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +import os +import tempfile + +from ome_zarr.cli import main + + +class TestCli: + @classmethod + def setup_class(cls): + cls.path = tempfile.TemporaryDirectory().name + + def test_coins(self): + filename = os.path.join(self.path, "coins") + main(["coins", filename]) + main(["info", filename]) diff --git a/tests/test_layer.py b/tests/test_layer.py new file mode 100644 index 00000000..1ed9d730 --- /dev/null +++ b/tests/test_layer.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import os +import tempfile + +from ome_zarr.data import create_zarr +from ome_zarr.io import parse_url +from ome_zarr.reader import Layer + + +class TestLayer: + @classmethod + def setup_class(cls): + cls.path = tempfile.TemporaryDirectory().name + create_zarr(cls.path) + + def test_image(self): + layer = Layer(parse_url(self.path)) + assert layer.data + assert layer.metadata + + def test_labels(self): + filename = os.path.join(self.path + "/labels") + layer = Layer(parse_url(filename)) + assert not layer.data + assert not layer.metadata + + def test_label(self): + filename = os.path.join(self.path + "/labels/coins") + layer = Layer(parse_url(filename)) + assert layer.data + assert layer.metadata diff --git a/tests/test_napari.py b/tests/test_napari.py new file mode 100644 index 00000000..1b49c2cc --- /dev/null +++ b/tests/test_napari.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +import os +import tempfile + +from ome_zarr.data import astronaut, create_zarr +from ome_zarr.napari import napari_get_reader + + +class TestNapari: + @classmethod + def setup_class(cls): + cls.path = tempfile.TemporaryDirectory().name + create_zarr(cls.path, astronaut) + + def test_image(self): + filename = os.path.join(self.path) + transform = napari_get_reader(filename) + for layer_data in transform(): + data, metadata = layer_data + assert data + assert metadata + assert 1 == metadata["channel_axis"] + assert ["Red", "Green", "Blue"] == metadata["name"] + assert [[0, 1], [0, 1], [0, 1]] == metadata["contrast_limits"] + assert [True, True, True] == metadata["visible"] + + def test_labels(self): + filename = os.path.join(self.path + "/labels") + transform = napari_get_reader(filename) + for layer_data in transform(): + data, metadata = layer_data + assert data + assert metadata + + def test_label(self): + filename = os.path.join(self.path + "/labels/coins") + transform = napari_get_reader(filename) + for layer_data in transform(): + data, metadata = layer_data + assert data + assert metadata diff --git a/tests/test_ome_zarr.py b/tests/test_ome_zarr.py index e73daff7..3aee9057 100644 --- a/tests/test_ome_zarr.py +++ b/tests/test_ome_zarr.py @@ -4,11 +4,10 @@ import os import tempfile +from ome_zarr.data import astronaut, create_zarr from ome_zarr.napari import napari_get_reader from ome_zarr.utils import download, info -from .create_test_data import create_zarr - def log_strings(idx, t, c, z, y, x, ct, cc, cz, cy, cx, dtype): yield f"resolution: {idx}" @@ -24,7 +23,7 @@ def setup_class(cls): usually contains tests). """ cls.path = tempfile.TemporaryDirectory(suffix=".zarr").name - create_zarr(cls.path) + create_zarr(cls.path, method=astronaut) def test_get_reader_hit(self): reader = napari_get_reader(self.path) @@ -58,10 +57,8 @@ def check_info_stdout(self, out): assert log in out # from info's print of omero metadata - assert "'channel_axis': 1" in out - assert "'name': ['Red', 'Green', 'Blue']" in out - assert "'contrast_limits': [[0, 1], [0, 1], [0, 1]]" in out - assert "'visible': [True, True, True]" in out + # note: some metadata is no longer handled by info but rather + # in the ome_zarr.napari.transform method def test_info(self, capsys, caplog): with caplog.at_level(logging.DEBUG): diff --git a/tests/test_reader.py b/tests/test_reader.py new file mode 100644 index 00000000..50c98d24 --- /dev/null +++ b/tests/test_reader.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +import os +import tempfile + +from ome_zarr.data import create_zarr +from ome_zarr.io import parse_url +from ome_zarr.reader import Layer, Reader + + +class TestReader: + @classmethod + def setup_class(cls): + cls.path = tempfile.TemporaryDirectory().name + create_zarr(cls.path) + + def assert_layer(self, layer: Layer): + if not layer.data or not layer.metadata: + assert False, f"Empty layer received: {layer}" + + def test_image(self): + reader = Reader(parse_url(self.path)) + for layer in reader(): + self.assert_layer(layer) + + def test_labels(self): + filename = os.path.join(self.path + "/labels") + reader = Reader(parse_url(filename)) + for layer in reader(): + self.assert_layer(layer) + + def test_label(self): + filename = os.path.join(self.path + "/labels/coins") + reader = Reader(parse_url(filename)) + for layer in reader(): + self.assert_layer(layer) diff --git a/tests/test_starting_points.py b/tests/test_starting_points.py new file mode 100644 index 00000000..0e4c4be9 --- /dev/null +++ b/tests/test_starting_points.py @@ -0,0 +1,45 @@ +import tempfile +from typing import Set, Type + +from ome_zarr.data import create_zarr +from ome_zarr.io import parse_url +from ome_zarr.reader import OMERO, Label, Labels, Layer, Multiscales, Spec + + +class TestStartingPoints: + """ + Creates a small but complete OME-Zarr file and tests that + readers will detect the correct type when starting at all + the various levels. + """ + + @classmethod + def setup_class(cls): + """ + """ + cls.path = tempfile.TemporaryDirectory(suffix=".zarr").name + create_zarr(cls.path) + + def matches(self, layer: Layer, expected: Set[Type[Spec]]): + found: Set[Type[Spec]] = set() + for spec in layer.specs: + found.add(type(spec)) + assert expected == found + + def test_top_level(self): + zarr = parse_url(self.path) + assert zarr is not None + layer = Layer(zarr) + self.matches(layer, set([Multiscales, OMERO])) + + def test_labels(self): + zarr = parse_url(self.path + "/labels") + assert zarr is not None + layer = Layer(zarr) + self.matches(layer, set([Labels])) + + def test_label(self): + zarr = parse_url(self.path + "/labels/coins") + assert zarr is not None + layer = Layer(zarr) + self.matches(layer, set([Label, Multiscales])) From 7c8cff1f6777561886dc7eb2734d9e2c9fbd5ca4 Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 25 Aug 2020 17:12:57 +0200 Subject: [PATCH 12/39] More tests and bug fixes --- .isort.cfg | 2 +- ome_zarr/cli.py | 23 +++++++++----- ome_zarr/io.py | 5 ++- ome_zarr/napari.py | 1 + ome_zarr/reader.py | 2 +- tests/test_cli.py | 18 ++++++----- tests/test_layer.py | 17 +++++------ tests/test_napari.py | 57 ++++++++++++++++++++--------------- tests/test_ome_zarr.py | 30 +++++++++--------- tests/test_reader.py | 17 +++++------ tests/test_starting_points.py | 42 +++++++++++++++++--------- 11 files changed, 124 insertions(+), 90 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 626bc53d..6b00c872 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -known_third_party = dask,numpy,requests,scipy,setuptools,skimage,vispy,zarr +known_third_party = dask,numpy,pytest,requests,scipy,setuptools,skimage,vispy,zarr diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index e0d34533..5cb5c873 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -5,7 +5,7 @@ import sys from typing import List -from .data import create_zarr as zarr_coins +from .data import astronaut, coins, create_zarr from .utils import download as zarr_download from .utils import info as zarr_info @@ -27,9 +27,15 @@ def download(args: argparse.Namespace) -> None: zarr_download(args.path, args.output, args.name) -def coins(args: argparse.Namespace) -> None: +def create(args: argparse.Namespace) -> None: config_logging(logging.INFO, args) - zarr_coins(args.path) + if args.method == "coins": + method = coins + elif args.method == "astronaut": + method = astronaut + else: + raise Exception(f"unknown method: {args.method}") + create_zarr(args.path, method=method) def main(args: List[str] = None) -> None: @@ -65,12 +71,15 @@ def main(args: List[str] = None) -> None: parser_download.set_defaults(func=download) # coin - parser_coins = subparsers.add_parser("coins") - parser_coins.add_argument("path") - parser_coins.set_defaults(func=coins) + parser_create = subparsers.add_parser("create") + parser_create.add_argument( + "--method", choices=("coins", "astronaut"), default="coins" + ) + parser_create.add_argument("path") + parser_create.set_defaults(func=create) if args is None: - ns = parser.parse_args(sys.argv) + ns = parser.parse_args(sys.argv[1:]) else: ns = parser.parse_args(args) ns.func(ns) diff --git a/ome_zarr/io.py b/ome_zarr/io.py index ca6d76d7..78f1afab 100644 --- a/ome_zarr/io.py +++ b/ome_zarr/io.py @@ -56,7 +56,10 @@ def load(self, subpath: str) -> da.core.Array: # TODO: update to from __future__ import annotations with 3.7+ def open(self, path: str) -> "BaseZarrLocation": """Create a new zarr for the given path""" - return self.__class__(posixpath.normpath(f"{self.zarr_path}/{path}")) + subpath = posixpath.join(self.zarr_path, path) + subpath = posixpath.normpath(subpath) + LOGGER.debug(f"open({self.__class__.__name__}({subpath}))") + return self.__class__(posixpath.normpath(f"{subpath}")) class LocalZarrLocation(BaseZarrLocation): diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 4ff1b878..7fe72a95 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -51,6 +51,7 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: results: List[LayerData] = list() for layer in layers: + LOGGER.debug(f"transforming {layer}") data = layer.data metadata = layer.metadata results.append((data, {"channel_axis": 1, **metadata})) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index fd759cc1..1dc9498f 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -153,7 +153,7 @@ def __init__(self, layer: Layer) -> None: { "visible": False, "name": name, - # "colormap": colors, + "colormap": colors, "metadata": {"image": self.lookup("image", {}), "path": name}, } ) diff --git a/tests/test_cli.py b/tests/test_cli.py index b3a59cd3..1190149d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,17 +1,21 @@ # -*- coding: utf-8 -*- -import os -import tempfile +import pytest from ome_zarr.cli import main class TestCli: - @classmethod - def setup_class(cls): - cls.path = tempfile.TemporaryDirectory().name + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") def test_coins(self): - filename = os.path.join(self.path, "coins") - main(["coins", filename]) + filename = str(self.path) + main(["create", "--method=coins", filename]) + main(["info", filename]) + + def test_astronaut(self): + filename = str(self.path) + main(["create", "--method=astronaut", filename]) main(["info", filename]) diff --git a/tests/test_layer.py b/tests/test_layer.py index 1ed9d730..c3f2c527 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -import os -import tempfile +import pytest from ome_zarr.data import create_zarr from ome_zarr.io import parse_url @@ -9,24 +8,24 @@ class TestLayer: - @classmethod - def setup_class(cls): - cls.path = tempfile.TemporaryDirectory().name - create_zarr(cls.path) + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") + create_zarr(str(self.path)) def test_image(self): - layer = Layer(parse_url(self.path)) + layer = Layer(parse_url(str(self.path))) assert layer.data assert layer.metadata def test_labels(self): - filename = os.path.join(self.path + "/labels") + filename = str(self.path.join("labels")) layer = Layer(parse_url(filename)) assert not layer.data assert not layer.metadata def test_label(self): - filename = os.path.join(self.path + "/labels/coins") + filename = str(self.path.join("labels", "coins")) layer = Layer(parse_url(filename)) assert layer.data assert layer.metadata diff --git a/tests/test_napari.py b/tests/test_napari.py index 1b49c2cc..3b876935 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -1,42 +1,51 @@ # -*- coding: utf-8 -*- -import os -import tempfile +import pytest from ome_zarr.data import astronaut, create_zarr from ome_zarr.napari import napari_get_reader class TestNapari: - @classmethod - def setup_class(cls): - cls.path = tempfile.TemporaryDirectory().name - create_zarr(cls.path, astronaut) + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") + create_zarr(str(self.path), astronaut) + + def assert_layer(self, layer_data): + data, metadata = layer_data + if not data or not metadata: + assert False, f"unknown layer: {layer_data}" + return data, metadata def test_image(self): - filename = os.path.join(self.path) - transform = napari_get_reader(filename) - for layer_data in transform(): - data, metadata = layer_data - assert data - assert metadata + layers = napari_get_reader(str(self.path))() + assert layers + for layer_data in layers: + data, metadata = self.assert_layer(layer_data) assert 1 == metadata["channel_axis"] assert ["Red", "Green", "Blue"] == metadata["name"] assert [[0, 1], [0, 1], [0, 1]] == metadata["contrast_limits"] assert [True, True, True] == metadata["visible"] def test_labels(self): - filename = os.path.join(self.path + "/labels") - transform = napari_get_reader(filename) - for layer_data in transform(): - data, metadata = layer_data - assert data - assert metadata + filename = str(self.path.join("labels")) + layers = napari_get_reader(filename)() + assert layers + for layer_data in layers: + data, metadata = self.assert_layer(layer_data) def test_label(self): - filename = os.path.join(self.path + "/labels/coins") - transform = napari_get_reader(filename) - for layer_data in transform(): - data, metadata = layer_data - assert data - assert metadata + filename = str(self.path.join("labels", "coins")) + layers = napari_get_reader(filename)() + assert layers + for layer_data in layers: + data, metadata = self.assert_layer(layer_data) + + def test_layers(self): + filename = str(self.path.join("labels", "coins")) + layers = napari_get_reader(filename)() + assert layers + # check order + # check name + # check visibility diff --git a/tests/test_ome_zarr.py b/tests/test_ome_zarr.py index 3aee9057..36956bf0 100644 --- a/tests/test_ome_zarr.py +++ b/tests/test_ome_zarr.py @@ -2,7 +2,8 @@ import logging import os -import tempfile + +import pytest from ome_zarr.data import astronaut, create_zarr from ome_zarr.napari import napari_get_reader @@ -17,22 +18,19 @@ def log_strings(idx, t, c, z, y, x, ct, cc, cz, cy, cx, dtype): class TestOmeZarr: - @classmethod - def setup_class(cls): - """ setup any state specific to the execution of the given class (which - usually contains tests). - """ - cls.path = tempfile.TemporaryDirectory(suffix=".zarr").name - create_zarr(cls.path, method=astronaut) + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") + create_zarr(str(self.path), method=astronaut) def test_get_reader_hit(self): - reader = napari_get_reader(self.path) + reader = napari_get_reader(str(self.path)) assert reader is not None assert callable(reader) def test_reader(self): - reader = napari_get_reader(self.path) - results = reader(self.path) + reader = napari_get_reader(str(self.path)) + results = reader(str(self.path)) assert results is not None and len(results) == 1 result = results[0] assert isinstance(result[0], list) @@ -42,7 +40,7 @@ def test_reader(self): def test_get_reader_with_list(self): # a better test here would use real data - reader = napari_get_reader([self.path]) + reader = napari_get_reader([str(self.path)]) assert reader is not None assert callable(reader) @@ -62,14 +60,14 @@ def check_info_stdout(self, out): def test_info(self, capsys, caplog): with caplog.at_level(logging.DEBUG): - info(self.path) + info(str(self.path)) self.check_info_stdout(caplog.text) - def test_download(self, capsys, caplog): - target = tempfile.TemporaryDirectory().name + def test_download(self, capsys, caplog, tmpdir): + target = tmpdir.mkdir("out") name = "test.zarr" with caplog.at_level(logging.DEBUG): - download(self.path, output_dir=target, zarr_name=name) + download(str(self.path), output_dir=target, zarr_name=name) download_zarr = os.path.join(target, name) assert os.path.exists(download_zarr) info(download_zarr) diff --git a/tests/test_reader.py b/tests/test_reader.py index 50c98d24..b38194c8 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -import os -import tempfile +import pytest from ome_zarr.data import create_zarr from ome_zarr.io import parse_url @@ -9,28 +8,28 @@ class TestReader: - @classmethod - def setup_class(cls): - cls.path = tempfile.TemporaryDirectory().name - create_zarr(cls.path) + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") + create_zarr(str(self.path)) def assert_layer(self, layer: Layer): if not layer.data or not layer.metadata: assert False, f"Empty layer received: {layer}" def test_image(self): - reader = Reader(parse_url(self.path)) + reader = Reader(parse_url(str(self.path))) for layer in reader(): self.assert_layer(layer) def test_labels(self): - filename = os.path.join(self.path + "/labels") + filename = str(self.path.join("labels")) reader = Reader(parse_url(filename)) for layer in reader(): self.assert_layer(layer) def test_label(self): - filename = os.path.join(self.path + "/labels/coins") + filename = str(self.path.join("labels", "coins")) reader = Reader(parse_url(filename)) for layer in reader(): self.assert_layer(layer) diff --git a/tests/test_starting_points.py b/tests/test_starting_points.py index 0e4c4be9..7b451b64 100644 --- a/tests/test_starting_points.py +++ b/tests/test_starting_points.py @@ -1,5 +1,6 @@ -import tempfile -from typing import Set, Type +from typing import List, Type + +import pytest from ome_zarr.data import create_zarr from ome_zarr.io import parse_url @@ -13,33 +14,44 @@ class TestStartingPoints: the various levels. """ - @classmethod - def setup_class(cls): - """ - """ - cls.path = tempfile.TemporaryDirectory(suffix=".zarr").name - create_zarr(cls.path) + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") + create_zarr(str(self.path)) + + def matches(self, layer: Layer, expected: List[Type[Spec]]): + found: List[Type[Spec]] = list() + for spec in layer.specs: + found.append(type(spec)) + + expected_names = sorted([x.__class__.__name__ for x in expected]) + found_names = sorted([x.__class__.__name__ for x in found]) + assert expected_names == found_names - def matches(self, layer: Layer, expected: Set[Type[Spec]]): - found: Set[Type[Spec]] = set() + def get_spec(self, layer: Layer, spec_type: Type[Spec]): for spec in layer.specs: - found.add(type(spec)) - assert expected == found + if isinstance(spec, spec_type): + return spec + assert False, f"no {spec_type} found" def test_top_level(self): - zarr = parse_url(self.path) + zarr = parse_url(str(self.path)) assert zarr is not None layer = Layer(zarr) self.matches(layer, set([Multiscales, OMERO])) + multiscales = self.get_spec(layer, Multiscales) + assert multiscales.lookup("multiscales", []) def test_labels(self): - zarr = parse_url(self.path + "/labels") + zarr = parse_url(str(self.path + "/labels")) assert zarr is not None layer = Layer(zarr) self.matches(layer, set([Labels])) def test_label(self): - zarr = parse_url(self.path + "/labels/coins") + zarr = parse_url(str(self.path + "/labels/coins")) assert zarr is not None layer = Layer(zarr) self.matches(layer, set([Label, Multiscales])) + multiscales = self.get_spec(layer, Multiscales) + assert multiscales.lookup("multiscales", []) From b57d4395b9cf6d8ee1439b6167f9e4d1e20c6c20 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 26 Aug 2020 08:17:02 +0200 Subject: [PATCH 13/39] Deal with empty labels from astronauts - replace test lookup for labels/coins - don't write labels group if no label exists --- ome_zarr/data.py | 39 +++++++++++++++++++++------------------ ome_zarr/io.py | 10 +++++++--- tests/test_napari.py | 6 +++--- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/ome_zarr/data.py b/ome_zarr/data.py index ac841ba4..86677827 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -64,7 +64,6 @@ def rgb_to_5d(pixels: np.ndarray) -> List: def write_multiscale(pyramid: List, group: zarr.Group) -> None: - paths = [] for path, dataset in enumerate(pyramid): group.create_dataset(str(path), data=pyramid[path]) @@ -75,7 +74,9 @@ def write_multiscale(pyramid: List, group: zarr.Group) -> None: def create_zarr( - zarr_directory: str, method: Callable[..., Tuple[List, List]] = coins + zarr_directory: str, + method: Callable[..., Tuple[List, List]] = coins, + label_name: str = "coins", ) -> None: pyramid, labels = method() @@ -84,9 +85,6 @@ def create_zarr( grp = zarr.group(store) write_multiscale(pyramid, grp) - labels_grp = grp.create_group("labels") - labels_grp.attrs["labels"] = ["coins"] - image_data = { "id": 1, "channels": [ @@ -113,16 +111,21 @@ def create_zarr( } grp.attrs["omero"] = image_data - coins_grp = labels_grp.create_group("coins") - write_multiscale(labels, coins_grp) - coins_grp.attrs["color"] = { - "1": rgba_to_int(50, 0, 0, 0), - "2": rgba_to_int(0, 50, 0, 0), - "3": rgba_to_int(0, 0, 50, 0), - "4": rgba_to_int(100, 0, 0, 0), - "5": rgba_to_int(0, 100, 0, 0), - "6": rgba_to_int(0, 0, 100, 0), - "7": rgba_to_int(50, 50, 50, 0), - "8": rgba_to_int(100, 100, 100, 0), - } - coins_grp.attrs["image"] = {"array": "../../", "source": {}} + if labels: + + labels_grp = grp.create_group("labels") + labels_grp.attrs["labels"] = [label_name] + + label_grp = labels_grp.create_group(label_name) + write_multiscale(labels, label_grp) + label_grp.attrs["color"] = { + "1": rgba_to_int(50, 0, 0, 0), + "2": rgba_to_int(0, 50, 0, 0), + "3": rgba_to_int(0, 0, 50, 0), + "4": rgba_to_int(100, 0, 0, 0), + "5": rgba_to_int(0, 100, 0, 0), + "6": rgba_to_int(0, 0, 100, 0), + "7": rgba_to_int(50, 50, 50, 0), + "8": rgba_to_int(100, 100, 100, 0), + } + label_grp.attrs["image"] = {"array": "../../", "source": {}} diff --git a/ome_zarr/io.py b/ome_zarr/io.py index 78f1afab..59b3325e 100644 --- a/ome_zarr/io.py +++ b/ome_zarr/io.py @@ -23,11 +23,11 @@ def __init__(self, path: str) -> None: self.zarr_path: str = path.endswith("/") and path or f"{path}/" self.zarray: JSONDict = self.get_json(".zarray") self.zgroup: JSONDict = self.get_json(".zgroup") - self.root_attrs: JSONDict = {} + self.__metadata: JSONDict = {} if self.zgroup: - self.root_attrs = self.get_json(".zattrs") + self.__metadata = self.get_json(".zattrs") elif self.zarray: - self.root_attrs = self.get_json(".zattrs") + self.__metadata = self.get_json(".zattrs") def __repr__(self) -> str: suffix = "" @@ -43,6 +43,10 @@ def exists(self) -> bool: def is_zarr(self) -> Optional[JSONDict]: return self.zarray or self.zgroup + @property + def root_attrs(self) -> JSONDict: + return dict(self.__metadata) + @abstractmethod def get_json(self, subpath: str) -> JSONDict: raise NotImplementedError("unknown") diff --git a/tests/test_napari.py b/tests/test_napari.py index 3b876935..50d3e3c6 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -10,7 +10,7 @@ class TestNapari: @pytest.fixture(autouse=True) def initdir(self, tmpdir): self.path = tmpdir.mkdir("data") - create_zarr(str(self.path), astronaut) + create_zarr(str(self.path), astronaut, "astronaut") def assert_layer(self, layer_data): data, metadata = layer_data @@ -36,14 +36,14 @@ def test_labels(self): data, metadata = self.assert_layer(layer_data) def test_label(self): - filename = str(self.path.join("labels", "coins")) + filename = str(self.path.join("labels", "astronaut")) layers = napari_get_reader(filename)() assert layers for layer_data in layers: data, metadata = self.assert_layer(layer_data) def test_layers(self): - filename = str(self.path.join("labels", "coins")) + filename = str(self.path.join("labels", "astronaut")) layers = napari_get_reader(filename)() assert layers # check order From 9665c4cbd8920d9ae0e8d0f1c620a5d6a2d5948f Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 26 Aug 2020 09:36:48 +0200 Subject: [PATCH 14/39] ome_zarr.scale: migrate scale.py to ome_zarr Provides the `ome_zarr scale` CLI command as well as the `ome_zarr.scale.Scaler` class for downsampling arrays as multiscales. originally: https://github.com/ome/omero-ms-zarr/pull/61 --- .isort.cfg | 7 +- ome_zarr/cli.py | 44 +++++++++++- ome_zarr/data.py | 51 ++++++++------ ome_zarr/scale.py | 167 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 247 insertions(+), 22 deletions(-) create mode 100644 ome_zarr/scale.py diff --git a/.isort.cfg b/.isort.cfg index 6b00c872..cbe16a06 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,7 @@ [settings] -known_third_party = dask,numpy,pytest,requests,scipy,setuptools,skimage,vispy,zarr +known_third_party = cv2,dask,numpy,pytest,requests,scipy,setuptools,skimage,vispy,zarr +multi_line_output=6 +include_trailing_comma=False +force_grid_wrap=0 +use_parentheses=True +line_length=120 diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index 5cb5c873..3ef9df52 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -6,6 +6,7 @@ from typing import List from .data import astronaut, coins, create_zarr +from .scale import Scaler from .utils import download as zarr_download from .utils import info as zarr_info @@ -31,11 +32,25 @@ def create(args: argparse.Namespace) -> None: config_logging(logging.INFO, args) if args.method == "coins": method = coins + label_name = "coins" elif args.method == "astronaut": method = astronaut + label_name = "circles" else: raise Exception(f"unknown method: {args.method}") - create_zarr(args.path, method=method) + create_zarr(args.path, method=method, label_name=label_name) + + +def scale(args: argparse.Namespace) -> None: + scaler = Scaler( + copy_metadata=args.copy_metadata, + downscale=args.downscale, + in_place=args.in_place, + labeled=args.labeled, + max_layer=args.max_layer, + method=args.method, + ) + scaler.scale(args.input_array, args.output_directory) def main(args: List[str] = None) -> None: @@ -70,7 +85,7 @@ def main(args: List[str] = None) -> None: parser_download.add_argument("--name", default="") parser_download.set_defaults(func=download) - # coin + # create parser_create = subparsers.add_parser("create") parser_create.add_argument( "--method", choices=("coins", "astronaut"), default="coins" @@ -78,6 +93,31 @@ def main(args: List[str] = None) -> None: parser_create.add_argument("path") parser_create.set_defaults(func=create) + parser_scale = subparsers.add_parser("scale") + parser_scale.add_argument("input_array") + parser_scale.add_argument("output_directory") + parser_scale.add_argument( + "--labeled", + action="store_true", + help="assert that the list of unique pixel values doesn't change", + ) + parser_scale.add_argument( + "--copy-metadata", + action="store_true", + help="copies the array metadata to the new group", + ) + parser_scale.add_argument( + "--method", choices=list(Scaler.methods()), default="nearest" + ) + parser_scale.add_argument( + "--in-place", action="store_true", help="if true, don't write the base array" + ) + parser_scale.add_argument("--downscale", type=int, default=2) + parser_scale.add_argument("--max_layer", type=int, default=4) + parser_scale.set_defaults(func=scale) + + ns = parser.parse_args() + if args is None: ns = parser.parse_args(sys.argv[1:]) else: diff --git a/ome_zarr/data.py b/ome_zarr/data.py index 86677827..e31954cb 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -9,9 +9,9 @@ from skimage.measure import label from skimage.morphology import closing, remove_small_objects, square from skimage.segmentation import clear_border -from skimage.transform import pyramid_gaussian from .conversions import rgba_to_int +from .scale import Scaler def coins() -> Tuple[List, List]: @@ -35,32 +35,45 @@ def coins() -> Tuple[List, List]: def astronaut() -> Tuple[List, List]: - base = np.tile(data.astronaut(), (2, 2, 1)) - gaussian = list(pyramid_gaussian(base, downscale=2, max_layer=4, multichannel=True)) - - pyramid = [] - # convert each level of pyramid into 5D image (t, c, z, y, x) - for pixels in gaussian: - red = pixels[:, :, 0] - green = pixels[:, :, 1] - blue = pixels[:, :, 2] - # wrap to make 5D: (t, c, z, y, x) - pixels = np.array([np.array([red]), np.array([green]), np.array([blue])]) - pixels = np.array([pixels]) - pyramid.append(pixels) - return pyramid, [] + scaler = Scaler() + + pixels = rgb_to_5d(np.tile(data.astronaut(), (2, 2, 1))) + pyramid = scaler.nearest(pixels) + + shape = list(pyramid[0].shape) + shape[1] = 1 + label = np.zeros(shape) + make_circle(100, 100, 1, label[0, 0, 0, 200:300, 200:300]) + make_circle(150, 150, 2, label[0, 0, 0, 250:400, 250:400]) + labels = scaler.nearest(label) + + return pyramid, labels + + +def make_circle(h: int, w: int, value: int, target: np.ndarray) -> None: + x = np.arange(0, w) + y = np.arange(0, h) + + cx = w // 2 + cy = h // 2 + r = min(w, h) // 2 + + mask = (x[np.newaxis, :] - cx) ** 2 + (y[:, np.newaxis] - cy) ** 2 < r ** 2 + target[mask] = value def rgb_to_5d(pixels: np.ndarray) -> List: """convert an RGB image into 5D image (t, c, z, y, x)""" if len(pixels.shape) == 2: - channels = [[np.array(pixels)]] + stack = np.array([pixels]) + channels = np.array([stack]) elif len(pixels.shape) == 3: - size_c = pixels.shape(2) - channels = [np.array(pixels[:, :, c]) for c in range(size_c)] + size_c = pixels.shape[2] + channels = [np.array([pixels[:, :, c]]) for c in range(size_c)] else: assert False, f"expecting 2 or 3d: ({pixels.shape})" - return [np.array(channels)] + video = np.array([channels]) + return video def write_multiscale(pyramid: List, group: zarr.Group) -> None: diff --git a/ome_zarr/scale.py b/ome_zarr/scale.py new file mode 100644 index 00000000..f58e180d --- /dev/null +++ b/ome_zarr/scale.py @@ -0,0 +1,167 @@ +import inspect +import logging +import os +from collections.abc import MutableMapping +from dataclasses import dataclass +from typing import Callable, Iterator, List + +import cv2 +import numpy as np +import zarr +from scipy.ndimage import zoom +from skimage.transform import downscale_local_mean, pyramid_gaussian, pyramid_laplacian + +LOGGER = logging.getLogger("ome_zarr.scale") + + +@dataclass +class Scaler: + + copy_metadata: bool = False + downscale: int = 2 + in_place: bool = False + labeled: bool = False + max_layer: int = 4 + method: str = "nearest" + + @staticmethod + def methods() -> Iterator[str]: + funcs = inspect.getmembers(Scaler, predicate=inspect.isfunction) + for name, func in funcs: + if name in ("methods", "scale"): + continue + if name.startswith("_"): + continue + yield name + + def scale(self, input_array: str, output_directory: str) -> None: + + func = getattr(self, self.method, None) + if not func: + raise Exception + + store = self._check_store(output_directory) + base = zarr.open_array(input_array) + pyramid = func(base) + + if self.labeled: + self._assert_values(pyramid) + + grp = self._create_group(store, base, pyramid) + + if self.copy_metadata: + print(f"copying attribute keys: {list(base.attrs.keys())}") + grp.attrs.update(base.attrs) + + def _check_store(self, output_directory: str) -> MutableMapping: + assert not os.path.exists(output_directory) + return zarr.DirectoryStore(output_directory) + + def _assert_values(self, pyramid: List[np.ndarray]) -> None: + expected = set(np.unique(pyramid[0])) + print(f"level 0 {pyramid[0].shape} = {len(expected)} labels") + for i in range(1, len(pyramid)): + level = pyramid[i] + print(f"level {i}", pyramid[i].shape, len(expected)) + found = set(np.unique(level)) + if not expected.issuperset(found): + raise Exception( + f"{len(found)} found values are not " + "a subset of {len(expected)} values" + ) + + def _create_group( + self, store: MutableMapping, base: np.ndarray, pyramid: List[np.ndarray] + ) -> zarr.hierarchy.Group: + grp = zarr.group(store) + grp.create_dataset("base", data=base) + series = [] + for i, dataset in enumerate(pyramid): + if i == 0: + path = "base" + else: + path = "%s" % i + grp.create_dataset(path, data=pyramid[i]) + series.append({"path": path}) + return grp + + # + # Scaling methods + # + + def nearest(self, base: np.ndarray) -> List[np.ndarray]: + def func(plane: np.ndarray, sizeY: int, sizeX: int) -> np.ndarray: + return cv2.resize( + plane, + dsize=(sizeY // self.downscale, sizeX // self.downscale), + interpolation=cv2.INTER_NEAREST, + ) + + return self._by_plane(base, func) + + def gaussian(self, base: np.ndarray) -> List[np.ndarray]: + return list( + pyramid_gaussian( + base, + downscale=self.downscale, + max_layer=self.max_layer, + multichannel=False, + ) + ) + + def laplacian(self, base: np.ndarray) -> List[np.ndarray]: + return list( + pyramid_laplacian( + base, + downscale=self.downscale, + max_layer=self.max_layer, + multichannel=False, + ) + ) + + def local_mean(self, base: np.ndarray) -> List[np.ndarray]: + # FIXME: fix hard-coding + rv = [base] + for i in range(self.max_layer): + rv.append( + downscale_local_mean( + rv[-1], factors=(1, 1, 1, self.downscale, self.downscale) + ) + ) + return rv + + def zoom(self, base: np.ndarray) -> List[np.ndarray]: + rv = [base] + print(base.shape) + for i in range(self.max_layer): + print(i, self.downscale) + rv.append(zoom(base, self.downscale ** i)) + print(rv[-1].shape) + return list(reversed(rv)) + + # + # Helpers + # + + def _by_plane( + self, base: np.ndarray, func: Callable[[np.ndarray, int, int], np.ndarray], + ) -> np.ndarray: + + assert 5 == len(base.shape) + + rv = [base] + for i in range(self.max_layer): + fiveD = rv[-1] + # FIXME: fix hard-coding of dimensions + T, C, Z, Y, X = fiveD.shape + + smaller = None + for t in range(T): + for c in range(C): + for z in range(Z): + out = func(fiveD[t][c][z][:], Y, X) + if smaller is None: + smaller = np.zeros((T, C, Z, out.shape[0], out.shape[1])) + smaller[t][c][z] = out + rv.append(smaller) + return rv From 6b1858186e95cf109666f022ddbd451762337c5c Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 26 Aug 2020 12:10:00 +0200 Subject: [PATCH 15/39] Enable make_test_viewer from napari --- setup.py | 1 + tests/test_napari.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/setup.py b/setup.py index 8ee34298..069ece2f 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ def read(fname): entry_points={ "console_scripts": ["ome_zarr = ome_zarr.cli:main"], "napari.plugin": ["ome_zarr = ome_zarr.napari"], + "pytest11": ["napari-conftest = napari.conftest"], }, extras_require={"napari": ["napari"]}, tests_require=["pytest", "pytest-capturelog"], diff --git a/tests/test_napari.py b/tests/test_napari.py index 50d3e3c6..f05eb272 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import numpy as np import pytest from ome_zarr.data import astronaut, create_zarr @@ -49,3 +50,21 @@ def test_layers(self): # check order # check name # check visibility + + def test_viewer(self, make_test_viewer): + """example of testing the viewer""" + viewer = make_test_viewer() + + shapes = [(4000, 3000), (2000, 1500), (1000, 750), (500, 375)] + np.random.seed(0) + data = [np.random.random(s) for s in shapes] + _ = viewer.add_image(data, multiscale=True, contrast_limits=[0, 1]) + layer = viewer.layers[0] + + # Set canvas size to target amount + viewer.window.qt_viewer.view.canvas.size = (800, 600) + list(viewer.window.qt_viewer.layer_to_visual.values())[0].on_draw(None) + + # Check that current level is first large enough to fill the canvas with + # a greater than one pixel depth + assert layer.data_level == 2 From 4fb42135b968b851841831e8db29edbe5d31b3e7 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 26 Aug 2020 15:20:11 +0200 Subject: [PATCH 16/39] Fix recursive reading, info, and downloading --- ome_zarr/cli.py | 9 ++-- ome_zarr/napari.py | 7 ++- ome_zarr/reader.py | 78 ++++++++++++++++++++---------- ome_zarr/utils.py | 115 +++++++++++++++++++++++++++------------------ 4 files changed, 132 insertions(+), 77 deletions(-) diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index 3ef9df52..0de6ccca 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -19,17 +19,17 @@ def config_logging(loglevel: int, args: argparse.Namespace) -> None: def info(args: argparse.Namespace) -> None: - config_logging(logging.INFO, args) - zarr_info(args.path) + config_logging(logging.WARN, args) + list(zarr_info(args.path)) def download(args: argparse.Namespace) -> None: config_logging(logging.WARN, args) - zarr_download(args.path, args.output, args.name) + zarr_download(args.path, args.output) def create(args: argparse.Namespace) -> None: - config_logging(logging.INFO, args) + config_logging(logging.WARN, args) if args.method == "coins": method = coins label_name = "coins" @@ -82,7 +82,6 @@ def main(args: List[str] = None) -> None: parser_download = subparsers.add_parser("download") parser_download.add_argument("path") parser_download.add_argument("--output", default="") - parser_download.add_argument("--name", default="") parser_download.set_defaults(func=download) # create diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 7fe72a95..9753bbd9 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -51,10 +51,13 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: results: List[LayerData] = list() for layer in layers: - LOGGER.debug(f"transforming {layer}") data = layer.data metadata = layer.metadata - results.append((data, {"channel_axis": 1, **metadata})) + if not data: + LOGGER.debug(f"skipping non-data {layer}") + else: + LOGGER.debug(f"transforming {layer}") + results.append((data, {"channel_axis": 1, **metadata})) return results return f diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 1dc9498f..65a3a94d 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -3,7 +3,6 @@ """ import logging -import posixpath from abc import ABC from typing import Any, Dict, Iterator, List, Optional, Union @@ -23,8 +22,10 @@ class Layer: the data hierarchy. """ - def __init__(self, zarr: BaseZarrLocation): + def __init__(self, zarr: BaseZarrLocation, root: Union["Layer", "Reader"]): self.zarr = zarr + self.root = root + self.seen: List[str] = root.seen self.visible = True # Likely to be updated by specs @@ -44,11 +45,30 @@ def __init__(self, zarr: BaseZarrLocation): if OMERO.matches(zarr): self.specs.append(OMERO(self)) + def add(self, zarr: BaseZarrLocation, prepend: bool = False,) -> "Optional[Layer]": + """ + Create a child layer if this location has not yet been seen; + otherwise return None + """ + + if zarr.zarr_path in self.seen: + LOGGER.debug(f"already seen {zarr}; stopping recursion") + return None + + self.seen.append(zarr.zarr_path) + layer = Layer(zarr, self) + if prepend: + self.pre_layers.append(layer) + else: + self.post_layers.append(layer) + + return layer + def write_metadata(self, metadata: JSONDict) -> None: for spec in self.specs: metadata.update(self.zarr.root_attrs) - def __str__(self) -> str: + def __repr__(self) -> str: suffix = "" if self.zarr.zgroup: suffix += " [zgroup]" @@ -97,8 +117,8 @@ def __init__(self, layer: Layer) -> None: label_names = self.lookup("labels", []) for name in label_names: child_zarr = self.zarr.open(name) - child_layer = Layer(child_zarr) - layer.post_layers.append(child_layer) + if child_zarr.exists(): + layer.add(child_zarr) class Label(Spec): @@ -119,17 +139,20 @@ def __init__(self, layer: Layer) -> None: super().__init__(layer) layer.visible = True - path = self.lookup("path", None) image = self.lookup("image", {}).get("array", None) - if path and image: + parent_zarr = None + if image: # This is an ome mask, load the image - parent = posixpath.normpath(f"{path}/{image}") - LOGGER.debug(f"delegating to parent image: {parent}") - parent_zarr = self.zarr.open(parent) + parent_zarr = self.zarr.open(image) if parent_zarr.exists(): - parent_layer = Layer(parent_zarr) - layer.pre_layers.append(parent_layer) - layer.visible = False + LOGGER.debug(f"delegating to parent image: {parent_zarr}") + parent_layer = layer.add(parent_zarr, prepend=True) + if parent_layer is not None: + layer.visible = False + else: + parent_zarr = None + if parent_zarr is None: + LOGGER.warn(f"no parent found for {self}: {image}") # Metadata: TODO move to a class colors: Dict[Union[int, bool], List[float]] = {} @@ -199,9 +222,8 @@ def __init__(self, layer: Layer) -> None: # Load possible layer data child_zarr = self.zarr.open("labels") - # Creating a layer propagates to sub-specs, but the layer itself - # should not be registered. - Layer(child_zarr) + if child_zarr.exists(): + layer.add(child_zarr) class OMERO(Spec): @@ -281,19 +303,14 @@ class Reader: def __init__(self, zarr: BaseZarrLocation) -> None: assert zarr.is_zarr() self.zarr = zarr + self.seen: List[str] = [zarr.zarr_path] def __call__(self) -> Iterator[Layer]: - layer = Layer(self.zarr) + layer = Layer(self.zarr, self) if layer.specs: # Something has matched - LOGGER.debug(f"treating {self.zarr} as ome-zarr") - # FIXME -- this will need recursion - for pre_layer in layer.pre_layers: - yield pre_layer - if layer.data: - yield layer - for post_layer in layer.post_layers: - yield post_layer + LOGGER.debug(f"treating {self.zarr} as ome-zarr") + yield from self.descend(layer) # TODO: API thoughts for the Spec type # - ask for earlier_layers, later_layers (i.e. priorities) @@ -310,3 +327,14 @@ def __call__(self) -> Iterator[Layer]: else: LOGGER.debug(f"ignoring {self.zarr}") # yield nothing + + def descend(self, layer: Layer, depth: int = 0) -> Iterator[Layer]: + + for pre_layer in layer.pre_layers: + yield from self.descend(pre_layer, depth + 1) + + LOGGER.debug(f"returning {layer}") + yield layer + + for post_layer in layer.post_layers: + yield from self.descend(post_layer, depth + 1) diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py index 421cba20..51186b41 100644 --- a/ome_zarr/utils.py +++ b/ome_zarr/utils.py @@ -5,74 +5,99 @@ import json import logging import os -from typing import List, Optional +from typing import Iterator, List import dask.array as da +import zarr from dask.diagnostics import ProgressBar from .io import parse_url -from .reader import OMERO, Layer, Multiscales +from .reader import Layer, Multiscales, Reader from .types import JSONDict LOGGER = logging.getLogger("ome_zarr.utils") -def info(path: str) -> Optional[Layer]: +def info(path: str) -> Iterator[Layer]: """ print information about the ome-zarr fileset """ zarr = parse_url(path) if not zarr: print(f"not a zarr: {zarr}") - return None else: - layer = Layer(zarr) - if not layer.specs: - print(f"not an ome-zarr: {zarr}") - LOGGER.debug(layer.data) - return layer + reader = Reader(zarr) + for layer in reader(): + if not layer.specs: + print(f"not an ome-zarr: {zarr}") + print(layer) + print(" - metadata") + for spec in layer.specs: + print(f" - {spec.__class__.__name__}") + print(" - data") + for array in layer.data: + print(f" - {array.shape}") + LOGGER.debug(layer.data) + yield layer -def download(path: str, output_dir: str = ".", zarr_name: str = "") -> None: +def download(path: str, output_dir: str = ".") -> None: """ download zarr from URL """ - layer = info(path) - if not layer: - return - image_id = "unknown" - resolutions: List[da.core.Array] = [] - datasets: List[str] = [] - for spec in layer.specs: - if isinstance(spec, OMERO): - image_id = spec.image_data.get("id", image_id) - if isinstance(spec, Multiscales): - datasets = spec.datasets - resolutions = layer.data - if not datasets or not resolutions: - print("no multiscales data found") - return - LOGGER.info("image_id %s", image_id) - if not zarr_name: - zarr_name = f"{image_id}.zarr" + location = parse_url(path) + if not location: + print(f"not a zarr: {location}") + else: + reader = Reader(location) + layers: List[Layer] = list() + paths: List[str] = list() + for layer in reader(): + layers.append(layer) + paths.append(layer.zarr.zarr_path) + + first_mismatch = -1 + min_length = min([len(x) for x in paths]) + for idx in range(min_length): + if len(set([x[idx] for x in paths])) == 1: + first_mismatch += 1 + else: + break + + if first_mismatch <= 0: + raise Exception("No common prefix") + print("downloading...") + for path in paths: + print(" ", path[first_mismatch - 1 :]) + + for layer, path in zip(layers, paths): - target_dir = os.path.join(output_dir, f"{zarr_name}") - if os.path.exists(target_dir): - print(f"{target_dir} already exists!") - return - print(f"downloading to {target_dir}") + target_dir = os.path.join(output_dir, f"{path}") + if os.path.exists(target_dir): + print(f"{target_dir} already exists!") + return + print(f"to {target_dir}") - pbar = ProgressBar() - for dataset, data in reversed(list(zip(datasets, resolutions))): - print("X", layer, dataset, data) - LOGGER.info(f"resolution {dataset}...") - with pbar: - data.to_zarr(os.path.join(target_dir, dataset)) + resolutions: List[da.core.Array] = [] + datasets: List[str] = [] + for spec in layer.specs: + if isinstance(spec, Multiscales): + datasets = spec.datasets + resolutions = layer.data + if datasets and resolutions: + pbar = ProgressBar() + for dataset, data in reversed(list(zip(datasets, resolutions))): + LOGGER.info(f"resolution {dataset}...") + with pbar: + data.to_zarr(os.path.join(target_dir, dataset)) + else: + # Assume a group that needs metadata, like labels + zarr.group(target_dir) - with open(os.path.join(target_dir, ".zgroup"), "w") as f: - f.write(json.dumps(layer.zarr.zgroup)) - with open(os.path.join(target_dir, ".zattrs"), "w") as f: - metadata: JSONDict = {} - layer.write_metadata(metadata) - f.write(json.dumps(metadata)) + with open(os.path.join(target_dir, ".zgroup"), "w") as f: + f.write(json.dumps(layer.zarr.zgroup)) + with open(os.path.join(target_dir, ".zattrs"), "w") as f: + metadata: JSONDict = {} + layer.write_metadata(metadata) + f.write(json.dumps(metadata)) From c4ef7389472d26a6fc3c8f7f84d40ede6f2b4a10 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 26 Aug 2020 17:46:34 +0200 Subject: [PATCH 17/39] Fix number of channels in coins() --- ome_zarr/data.py | 55 +++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/ome_zarr/data.py b/ome_zarr/data.py index e31954cb..ad074686 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -13,6 +13,8 @@ from .conversions import rgba_to_int from .scale import Scaler +CHANNEL_DIMENSION = 1 + def coins() -> Tuple[List, List]: """ @@ -98,30 +100,35 @@ def create_zarr( grp = zarr.group(store) write_multiscale(pyramid, grp) - image_data = { - "id": 1, - "channels": [ - { - "color": "FF0000", - "window": {"start": 0, "end": 1}, - "label": "Red", - "active": True, - }, - { - "color": "00FF00", - "window": {"start": 0, "end": 1}, - "label": "Green", - "active": True, - }, - { - "color": "0000FF", - "window": {"start": 0, "end": 1}, - "label": "Blue", - "active": True, - }, - ], - "rdefs": {"model": "color"}, - } + if pyramid[0].shape[CHANNEL_DIMENSION] == 1: + image_data = { + "channels": [{"window": {"start": 0, "end": 1}}], + "rdefs": {"model": "grayscale"}, + } + else: + image_data = { + "channels": [ + { + "color": "FF0000", + "window": {"start": 0, "end": 1}, + "label": "Red", + "active": True, + }, + { + "color": "00FF00", + "window": {"start": 0, "end": 1}, + "label": "Green", + "active": True, + }, + { + "color": "0000FF", + "window": {"start": 0, "end": 1}, + "label": "Blue", + "active": True, + }, + ], + "rdefs": {"model": "color"}, + } grp.attrs["omero"] = image_data if labels: From 77e631aeeb580b3640634a0b398e2d6b941b8640 Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 27 Aug 2020 16:53:44 +0200 Subject: [PATCH 18/39] Fix CLI tools incl. prefix stripping --- ome_zarr/cli.py | 11 +++- ome_zarr/reader.py | 12 +++- ome_zarr/utils.py | 107 +++++++++++++++++++--------------- tests/test_cli.py | 55 +++++++++++++++-- tests/test_layer.py | 6 +- tests/test_napari.py | 17 +++--- tests/test_ome_zarr.py | 23 ++++---- tests/test_reader.py | 11 ++-- tests/test_starting_points.py | 8 +-- 9 files changed, 158 insertions(+), 92 deletions(-) diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index 0de6ccca..b3a0f7d0 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -81,7 +81,7 @@ def main(args: List[str] = None) -> None: # download parser_download = subparsers.add_parser("download") parser_download.add_argument("path") - parser_download.add_argument("--output", default="") + parser_download.add_argument("--output", default=".") parser_download.set_defaults(func=download) # create @@ -115,10 +115,15 @@ def main(args: List[str] = None) -> None: parser_scale.add_argument("--max_layer", type=int, default=4) parser_scale.set_defaults(func=scale) - ns = parser.parse_args() + ns = parser.parse_args(args) if args is None: ns = parser.parse_args(sys.argv[1:]) else: ns = parser.parse_args(args) - ns.func(ns) + + try: + ns.func(ns) + except AssertionError as error: + logging.getLogger("ome_zarr.cli").error(error) + sys.exit(2) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 65a3a94d..c0f0855c 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -4,7 +4,7 @@ import logging from abc import ABC -from typing import Any, Dict, Iterator, List, Optional, Union +from typing import Any, Dict, Iterator, List, Optional, Union, cast import dask.array as da from vispy.color import Colormap @@ -22,10 +22,16 @@ class Layer: the data hierarchy. """ - def __init__(self, zarr: BaseZarrLocation, root: Union["Layer", "Reader"]): + def __init__( + self, zarr: BaseZarrLocation, root: Union["Layer", "Reader", List[str]] + ): self.zarr = zarr self.root = root - self.seen: List[str] = root.seen + self.seen: List[str] = [] + if isinstance(root, Layer) or isinstance(root, Reader): + self.seen = root.seen + else: + self.seen = cast(List[str], root) self.visible = True # Likely to be updated by specs diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py index 51186b41..fbc38e19 100644 --- a/ome_zarr/utils.py +++ b/ome_zarr/utils.py @@ -23,62 +23,50 @@ def info(path: str) -> Iterator[Layer]: print information about the ome-zarr fileset """ zarr = parse_url(path) - if not zarr: - print(f"not a zarr: {zarr}") - else: - reader = Reader(zarr) - for layer in reader(): - if not layer.specs: - print(f"not an ome-zarr: {zarr}") - print(layer) - print(" - metadata") - for spec in layer.specs: - print(f" - {spec.__class__.__name__}") - print(" - data") - for array in layer.data: - print(f" - {array.shape}") - LOGGER.debug(layer.data) - yield layer - - -def download(path: str, output_dir: str = ".") -> None: + assert zarr, f"not a zarr: {zarr}" + reader = Reader(zarr) + for layer in reader(): + + if not layer.specs: + print(f"not an ome-zarr: {zarr}") + continue + + print(layer) + print(" - metadata") + for spec in layer.specs: + print(f" - {spec.__class__.__name__}") + print(" - data") + for array in layer.data: + print(f" - {array.shape}") + LOGGER.debug(layer.data) + yield layer + + +def download(input_path: str, output_dir: str = ".") -> None: """ download zarr from URL """ - location = parse_url(path) - if not location: - print(f"not a zarr: {location}") - else: - reader = Reader(location) - layers: List[Layer] = list() - paths: List[str] = list() - for layer in reader(): - layers.append(layer) - paths.append(layer.zarr.zarr_path) - - first_mismatch = -1 - min_length = min([len(x) for x in paths]) - for idx in range(min_length): - if len(set([x[idx] for x in paths])) == 1: - first_mismatch += 1 - else: - break + location = parse_url(input_path) + assert location, f"not a zarr: {location}" - if first_mismatch <= 0: - raise Exception("No common prefix") - print("downloading...") - for path in paths: - print(" ", path[first_mismatch - 1 :]) + reader = Reader(location) + layers: List[Layer] = list() + paths: List[str] = list() + for layer in reader(): + layers.append(layer) + paths.append(layer.zarr.zarr_path) - for layer, path in zip(layers, paths): + strip_common_prefix(paths) - target_dir = os.path.join(output_dir, f"{path}") - if os.path.exists(target_dir): - print(f"{target_dir} already exists!") - return - print(f"to {target_dir}") + assert not os.path.exists(output_dir), f"{output_dir} already exists!" + print("downloading...") + for path in paths: + print(" ", path) + print(f"to {output_dir}") + for path, layer in sorted(zip(paths, layers)): + target_dir = os.path.join(output_dir, f"{path}") resolutions: List[da.core.Array] = [] datasets: List[str] = [] for spec in layer.specs: @@ -101,3 +89,26 @@ def download(path: str, output_dir: str = ".") -> None: metadata: JSONDict = {} layer.write_metadata(metadata) f.write(json.dumps(metadata)) + + +def strip_common_prefix(paths: List[str]) -> None: + parts: List[List[str]] = [x.split(os.path.sep) for x in paths] + + first_mismatch = 0 + min_length = min([len(x) for x in parts]) + + for idx in range(min_length): + if len(set([x[idx] for x in parts])) == 1: + first_mismatch += 1 + else: + break + + if first_mismatch <= 0: + msg = "No common prefix:\n" + for path in parts: + msg += f"{path}\n" + raise Exception(msg) + + for idx, path in enumerate(parts): + base = os.path.sep.join(path[first_mismatch - 1 :]) + paths[idx] = base diff --git a/tests/test_cli.py b/tests/test_cli.py index 1190149d..4535e726 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,21 +1,66 @@ # -*- coding: utf-8 -*- +import os +from collections import deque +from pathlib import Path +from typing import Sequence + import pytest from ome_zarr.cli import main +from ome_zarr.utils import strip_common_prefix class TestCli: @pytest.fixture(autouse=True) def initdir(self, tmpdir): - self.path = tmpdir.mkdir("data") + self.path = (tmpdir / "data").mkdir() - def test_coins(self): - filename = str(self.path) + def test_coins_info(self): + filename = str(self.path) + "-1" main(["create", "--method=coins", filename]) main(["info", filename]) - def test_astronaut(self): - filename = str(self.path) + def test_astronaut_info(self): + filename = str(self.path) + "-2" main(["create", "--method=astronaut", filename]) main(["info", filename]) + + def test_astronaut_download(self, tmpdir): + out = str(tmpdir / "out") + filename = str(self.path) + "-3" + basename = os.path.split(filename)[-1] + main(["create", "--method=astronaut", filename]) + main(["download", filename, f"--output={out}"]) + main(["info", f"{out}/{basename}"]) + + def test_strip_prefix_relative(self): + top = Path(".") / "d" + mid = Path(".") / "d" / "e" + bot = Path(".") / "d" / "e" / "f" + self._rotate_and_test(top, mid, bot) + + def test_strip_prefix_absolute(self): + top = Path("/") / "a" / "b" / "c" / "d" + mid = Path("/") / "a" / "b" / "c" / "d" / "e" + bot = Path("/") / "a" / "b" / "c" / "d" / "e" / "f" + self._rotate_and_test(top, mid, bot) + + def _rotate_and_test(self, *hierarchy: Path, reverse: bool = True): + results: Sequence[str] = ( + str(Path("d")), + str(Path("d") / "e"), + str(Path("d") / "e" / "f"), + ) + for x in range(3): + firstpass = deque(hierarchy) + firstpass.rotate(1) + + copy = [str(x) for x in firstpass] + strip_common_prefix(copy) + assert set(copy) == set(results) + + if reverse: + secondpass: deque = deque(hierarchy) + secondpass.reverse() + self._rotate_and_test(*list(secondpass), reverse=False) diff --git a/tests/test_layer.py b/tests/test_layer.py index c3f2c527..5034d95e 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -14,18 +14,18 @@ def initdir(self, tmpdir): create_zarr(str(self.path)) def test_image(self): - layer = Layer(parse_url(str(self.path))) + layer = Layer(parse_url(str(self.path)), list()) assert layer.data assert layer.metadata def test_labels(self): filename = str(self.path.join("labels")) - layer = Layer(parse_url(filename)) + layer = Layer(parse_url(filename), list()) assert not layer.data assert not layer.metadata def test_label(self): filename = str(self.path.join("labels", "coins")) - layer = Layer(parse_url(filename)) + layer = Layer(parse_url(filename), list()) assert layer.data assert layer.metadata diff --git a/tests/test_napari.py b/tests/test_napari.py index f05eb272..2cfa5a3a 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -21,13 +21,16 @@ def assert_layer(self, layer_data): def test_image(self): layers = napari_get_reader(str(self.path))() - assert layers - for layer_data in layers: - data, metadata = self.assert_layer(layer_data) - assert 1 == metadata["channel_axis"] - assert ["Red", "Green", "Blue"] == metadata["name"] - assert [[0, 1], [0, 1], [0, 1]] == metadata["contrast_limits"] - assert [True, True, True] == metadata["visible"] + assert len(layers) == 2 + image, label = layers + + data, metadata = self.assert_layer(image) + assert 1 == metadata["channel_axis"] + assert ["Red", "Green", "Blue"] == metadata["name"] + assert [[0, 1], [0, 1], [0, 1]] == metadata["contrast_limits"] + assert [True, True, True] == metadata["visible"] + + data, metadata = self.assert_layer(label) def test_labels(self): filename = str(self.path.join("labels")) diff --git a/tests/test_ome_zarr.py b/tests/test_ome_zarr.py index 36956bf0..82cd4dc1 100644 --- a/tests/test_ome_zarr.py +++ b/tests/test_ome_zarr.py @@ -31,12 +31,12 @@ def test_get_reader_hit(self): def test_reader(self): reader = napari_get_reader(str(self.path)) results = reader(str(self.path)) - assert results is not None and len(results) == 1 - result = results[0] - assert isinstance(result[0], list) - assert isinstance(result[1], dict) - assert result[1]["channel_axis"] == 1 - assert result[1]["name"] == ["Red", "Green", "Blue"] + assert len(results) == 2 + image, label = results + assert isinstance(image[0], list) + assert isinstance(image[1], dict) + assert image[1]["channel_axis"] == 1 + assert image[1]["name"] == ["Red", "Green", "Blue"] def test_get_reader_with_list(self): # a better test here would use real data @@ -58,17 +58,16 @@ def check_info_stdout(self, out): # note: some metadata is no longer handled by info but rather # in the ome_zarr.napari.transform method - def test_info(self, capsys, caplog): + def test_info(self, caplog): with caplog.at_level(logging.DEBUG): - info(str(self.path)) + list(info(str(self.path))) self.check_info_stdout(caplog.text) def test_download(self, capsys, caplog, tmpdir): - target = tmpdir.mkdir("out") - name = "test.zarr" + target = str(tmpdir / "out") with caplog.at_level(logging.DEBUG): - download(str(self.path), output_dir=target, zarr_name=name) - download_zarr = os.path.join(target, name) + download(str(self.path), output_dir=target) + download_zarr = os.path.join(target, "data") assert os.path.exists(download_zarr) info(download_zarr) self.check_info_stdout(caplog.text) diff --git a/tests/test_reader.py b/tests/test_reader.py index b38194c8..db6fc4e2 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -14,22 +14,19 @@ def initdir(self, tmpdir): create_zarr(str(self.path)) def assert_layer(self, layer: Layer): - if not layer.data or not layer.metadata: + if not layer.data and not layer.metadata: assert False, f"Empty layer received: {layer}" def test_image(self): reader = Reader(parse_url(str(self.path))) - for layer in reader(): - self.assert_layer(layer) + assert len(list(reader())) == 3 def test_labels(self): filename = str(self.path.join("labels")) reader = Reader(parse_url(filename)) - for layer in reader(): - self.assert_layer(layer) + assert len(list(reader())) == 3 def test_label(self): filename = str(self.path.join("labels", "coins")) reader = Reader(parse_url(filename)) - for layer in reader(): - self.assert_layer(layer) + assert len(list(reader())) == 3 diff --git a/tests/test_starting_points.py b/tests/test_starting_points.py index 7b451b64..0ee1c619 100644 --- a/tests/test_starting_points.py +++ b/tests/test_starting_points.py @@ -37,21 +37,21 @@ def get_spec(self, layer: Layer, spec_type: Type[Spec]): def test_top_level(self): zarr = parse_url(str(self.path)) assert zarr is not None - layer = Layer(zarr) - self.matches(layer, set([Multiscales, OMERO])) + layer = Layer(zarr, list()) + self.matches(layer, {Multiscales, OMERO}) multiscales = self.get_spec(layer, Multiscales) assert multiscales.lookup("multiscales", []) def test_labels(self): zarr = parse_url(str(self.path + "/labels")) assert zarr is not None - layer = Layer(zarr) + layer = Layer(zarr, list()) self.matches(layer, set([Labels])) def test_label(self): zarr = parse_url(str(self.path + "/labels/coins")) assert zarr is not None - layer = Layer(zarr) + layer = Layer(zarr, list()) self.matches(layer, set([Label, Multiscales])) multiscales = self.get_spec(layer, Multiscales) assert multiscales.lookup("multiscales", []) From 7a4f865db09e9edfb3cf8e587771dbc8c72b298f Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 28 Aug 2020 13:43:42 +0200 Subject: [PATCH 19/39] Update style and fill out well-formed docs --- .bumpversion.cfg | 4 +- .pre-commit-config.yaml | 38 ++++++++++++++++- ome_zarr/cli.py | 17 ++++++-- ome_zarr/conversions.py | 26 +++++++++++- ome_zarr/data.py | 27 +++++++++--- ome_zarr/io.py | 53 +++++++++++++++++++---- ome_zarr/napari.py | 9 ++-- ome_zarr/reader.py | 55 ++++++++++-------------- ome_zarr/scale.py | 79 ++++++++++++++++++++++++++--------- ome_zarr/types.py | 4 +- ome_zarr/utils.py | 28 +++++++++---- setup.py | 1 - tests/test_cli.py | 2 - tests/test_layer.py | 2 - tests/test_napari.py | 4 +- tests/test_ome_zarr.py | 2 - tests/test_reader.py | 2 - tests/test_starting_points.py | 11 ++--- 18 files changed, 251 insertions(+), 113 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index dd26a87e..a7b0e196 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -4,14 +4,14 @@ commit = True tag = True sign_tags = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? -serialize = +serialize = {major}.{minor}.{patch}.{release}{build} {major}.{minor}.{patch} [bumpversion:part:release] optional_value = prod first_value = dev -values = +values = dev prod diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 158770aa..b46ad1db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,15 +17,48 @@ repos: - id: black args: [--target-version=py36] + - repo: https://github.com/asottile/pyupgrade + rev: v2.7.2 + hooks: + - id: pyupgrade + args: + - --py36-plus + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v1.2.3 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-json + files: \.(json)$ + - id: check-yaml + - id: fix-encoding-pragma + args: + - --remove + - id: trailing-whitespace + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: pretty-format-json + args: + - --autofix + - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: - id: flake8 + additional_dependencies: [ + flake8-blind-except, + flake8-builtins, + flake8-rst-docstrings, + flake8-logging-format, + ] args: [ # default black line length is 88 - --max-line-length=88, + "--max-line-length=88", # Conflicts with black: E203 whitespace before ':' - --ignore=E203, + "--ignore=E203", + "--rst-roles=class,func,ref,module,const", ] - repo: https://github.com/pre-commit/mirrors-mypy @@ -48,3 +81,4 @@ repos: hooks: - id: yamllint # args: [--config-data=relaxed] + # diff --git a/ome_zarr/cli.py b/ome_zarr/cli.py index b3a0f7d0..6b1d59b3 100755 --- a/ome_zarr/cli.py +++ b/ome_zarr/cli.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python - +"""Entrypoint for the `ome_zarr` command-line tool.""" import argparse import logging import sys @@ -12,6 +11,11 @@ def config_logging(loglevel: int, args: argparse.Namespace) -> None: + """Configure logging taking the `verbose` and `quiet` arguments into account. + + Each `-v` increases the `loglevel` by 10 and each `-q` reduces the loglevel by 10. + For example, an initial loglevel of `INFO` will be converted to `DEBUG` via `-qqv`. + """ loglevel = loglevel - (10 * args.verbose) + (10 * args.quiet) logging.basicConfig(level=loglevel) # DEBUG logging for s3fs so we can track remote calls @@ -19,16 +23,22 @@ def config_logging(loglevel: int, args: argparse.Namespace) -> None: def info(args: argparse.Namespace) -> None: + """Wrap the :func:`~ome_zarr.utils.info` method.""" config_logging(logging.WARN, args) list(zarr_info(args.path)) def download(args: argparse.Namespace) -> None: + """Wrap the :func:`~ome_zarr.utils.download` method.""" config_logging(logging.WARN, args) zarr_download(args.path, args.output) def create(args: argparse.Namespace) -> None: + """Chooses between data generation methods in :module:`ome_zarr.utils` like. + + :func:`~ome_zarr.data.coins` or :func:`~ome_zarr.data.astronaut`. + """ config_logging(logging.WARN, args) if args.method == "coins": method = coins @@ -42,6 +52,7 @@ def create(args: argparse.Namespace) -> None: def scale(args: argparse.Namespace) -> None: + """Wrap the :func:`~ome_zarr.scale.Scaler.scale` method.""" scaler = Scaler( copy_metadata=args.copy_metadata, downscale=args.downscale, @@ -54,7 +65,7 @@ def scale(args: argparse.Namespace) -> None: def main(args: List[str] = None) -> None: - + """Run appropriate function with argparse arguments, handling errors.""" parser = argparse.ArgumentParser() parser.add_argument( "-v", diff --git a/ome_zarr/conversions.py b/ome_zarr/conversions.py index a13ff3a1..b7096ba4 100644 --- a/ome_zarr/conversions.py +++ b/ome_zarr/conversions.py @@ -1,10 +1,34 @@ +"""Simple conversion helpers.""" + from typing import List def int_to_rgba(v: int) -> List[float]: - """Get rgba (0-1) e.g. (1, 0.5, 0, 1) from integer""" + """Get rgba (0-1) e.g. (1, 0.5, 0, 1) from integer. + >>> print(int_to_rgba(0)) + [0.0, 0.0, 0.0, 0.0] + >>> print([round(x, 3) for x in int_to_rgba(100100)]) + [0.0, 0.004, 0.529, 0.016] + """ return [x / 255 for x in v.to_bytes(4, signed=True, byteorder="big")] +def int_to_rgba_255(v: int) -> List[int]: + """Get rgba (0-255) from integer. + >>> print(int_to_rgba_255(0)) + [0, 0, 0, 0] + >>> print([round(x, 3) for x in int_to_rgba_255(100100)]) + [0, 1, 135, 4] + """ + return [x for x in v.to_bytes(4, signed=True, byteorder="big")] + + def rgba_to_int(r: int, g: int, b: int, a: int) -> int: + """Use int.from_bytes to convert a color tuple. + + >>> print(rgba_to_int(0, 0, 0, 0)) + 0 + >>> print(rgba_to_int(0, 1, 135, 4)) + 100100 + """ return int.from_bytes([r, g, b, a], byteorder="big", signed=True) diff --git a/ome_zarr/data.py b/ome_zarr/data.py index ad074686..bd2416cf 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +"""Functions for generating synthetic data.""" from typing import Callable, List, Tuple import numpy as np @@ -17,9 +17,7 @@ def coins() -> Tuple[List, List]: - """ - Sample data from skimage - """ + """Sample data from skimage.""" # Thanks to Juan # https://gist.github.com/jni/62e07ddd135dbb107278bc04c0f9a8e7 image = data.coins()[50:-50, 50:-50] @@ -37,6 +35,7 @@ def coins() -> Tuple[List, List]: def astronaut() -> Tuple[List, List]: + """Sample data from skimage.""" scaler = Scaler() pixels = rgb_to_5d(np.tile(data.astronaut(), (2, 2, 1))) @@ -53,6 +52,21 @@ def astronaut() -> Tuple[List, List]: def make_circle(h: int, w: int, value: int, target: np.ndarray) -> None: + """Apply a 2D circular mask to the given array. + + >>> import numpy as np + >>> example = np.zeros((8, 8)) + >>> make_circle(8, 8, 1, example) + >>> print(example) + [[0. 0. 0. 0. 0. 0. 0. 0.] + [0. 0. 1. 1. 1. 1. 1. 0.] + [0. 1. 1. 1. 1. 1. 1. 1.] + [0. 1. 1. 1. 1. 1. 1. 1.] + [0. 1. 1. 1. 1. 1. 1. 1.] + [0. 1. 1. 1. 1. 1. 1. 1.] + [0. 1. 1. 1. 1. 1. 1. 1.] + [0. 0. 1. 1. 1. 1. 1. 0.]] + """ x = np.arange(0, w) y = np.arange(0, h) @@ -65,7 +79,7 @@ def make_circle(h: int, w: int, value: int, target: np.ndarray) -> None: def rgb_to_5d(pixels: np.ndarray) -> List: - """convert an RGB image into 5D image (t, c, z, y, x)""" + """Convert an RGB image into 5D image (t, c, z, y, x).""" if len(pixels.shape) == 2: stack = np.array([pixels]) channels = np.array([stack]) @@ -79,6 +93,7 @@ def rgb_to_5d(pixels: np.ndarray) -> List: def write_multiscale(pyramid: List, group: zarr.Group) -> None: + """Write a pyramid with multiscale metadata to disk.""" paths = [] for path, dataset in enumerate(pyramid): group.create_dataset(str(path), data=pyramid[path]) @@ -93,7 +108,7 @@ def create_zarr( method: Callable[..., Tuple[List, List]] = coins, label_name: str = "coins", ) -> None: - + """Generate a synthetic image pyramid with labels.""" pyramid, labels = method() store = zarr.DirectoryStore(zarr_directory) diff --git a/ome_zarr/io.py b/ome_zarr/io.py index 59b3325e..cd3719a6 100644 --- a/ome_zarr/io.py +++ b/ome_zarr/io.py @@ -1,5 +1,6 @@ -""" -Reading logic for ome-zarr +"""Reading logic for ome-zarr. + +Primary entry point is the :func:`~ome_zarr.io.parse_url` method. """ import json @@ -19,17 +20,28 @@ class BaseZarrLocation(ABC): + """ + Base IO primitive for reading Zarr data. + + No assumptions about the existence of the given path string are made. + Attempts are made to load various metadata files and cache them internally. + """ + def __init__(self, path: str) -> None: self.zarr_path: str = path.endswith("/") and path or f"{path}/" self.zarray: JSONDict = self.get_json(".zarray") self.zgroup: JSONDict = self.get_json(".zgroup") self.__metadata: JSONDict = {} + self.__exists: bool = True if self.zgroup: self.__metadata = self.get_json(".zattrs") elif self.zarray: self.__metadata = self.get_json(".zattrs") + else: + self.__exists = False def __repr__(self) -> str: + """Print the path as well as whether this is a group or an array.""" suffix = "" if self.zgroup: suffix += " [zgroup]" @@ -38,28 +50,30 @@ def __repr__(self) -> str: return f"{self.zarr_path}{suffix}" def exists(self) -> bool: - return os.path.exists(self.zarr_path) + """Return true if zgroup or zarray metadata exists.""" + return self.__exists def is_zarr(self) -> Optional[JSONDict]: + """Return true if either zarray or zgroup metadata exists.""" return self.zarray or self.zgroup @property def root_attrs(self) -> JSONDict: + """Return the contents of the zattrs file.""" return dict(self.__metadata) @abstractmethod def get_json(self, subpath: str) -> JSONDict: + """Must be implemented by subclasses.""" raise NotImplementedError("unknown") def load(self, subpath: str) -> da.core.Array: - """ - Use dask.array.from_zarr to load the subpath - """ + """Use dask.array.from_zarr to load the subpath.""" return da.from_zarr(f"{self.zarr_path}{subpath}") # TODO: update to from __future__ import annotations with 3.7+ - def open(self, path: str) -> "BaseZarrLocation": - """Create a new zarr for the given path""" + def create(self, path: str) -> "BaseZarrLocation": + """Create a new Zarr location for the given path.""" subpath = posixpath.join(self.zarr_path, path) subpath = posixpath.normpath(subpath) LOGGER.debug(f"open({self.__class__.__name__}({subpath}))") @@ -67,10 +81,21 @@ def open(self, path: str) -> "BaseZarrLocation": class LocalZarrLocation(BaseZarrLocation): + """ + Uses the :module:`json` library for loading JSON from disk. + """ + def get_json(self, subpath: str) -> JSONDict: + """ + Load and return a given subpath of self.zarr_path as JSON. + + If a file does not exist, an empty response is returned rather + than an exception. + """ filename = os.path.join(self.zarr_path, subpath) if not os.path.exists(filename): + LOGGER.debug(f"{filename} does not exist") return {} with open(filename) as f: @@ -78,7 +103,16 @@ def get_json(self, subpath: str) -> JSONDict: class RemoteZarrLocation(BaseZarrLocation): + """ Uses the :module:`requests` library for accessing Zarr metadata files. """ + def get_json(self, subpath: str) -> JSONDict: + """ + Load and return a given subpath of self.zarr_path as JSON. + + HTTP 403 and 404 responses are treated as if the file does not exist. + Exceptions during the remote connection are logged at the WARN level. + All other exceptions log at the ERROR level. + """ url = f"{self.zarr_path}{subpath}" try: rsp = requests.get(url) @@ -96,7 +130,8 @@ def get_json(self, subpath: str) -> JSONDict: def parse_url(path: str) -> Optional[BaseZarrLocation]: - """ convert a path string or URL to a BaseZarrLocation instance + """Convert a path string or URL to a BaseZarrLocation subclass. + >>> parse_url('does-not-exist') """ # Check is path is local directory first diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 9753bbd9..642ed3e2 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -1,8 +1,6 @@ -""" -This module is a napari plugin. +"""This module is a napari plugin. -It implements the ``napari_get_reader`` hook specification, (to create -a reader plugin). +It implements the ``napari_get_reader`` hook specification, (to create a reader plugin). """ @@ -29,8 +27,7 @@ def napari_hook_implementation( @napari_hook_implementation def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: - """ - Returns a reader for supported paths that include IDR ID + """Returns a reader for supported paths that include IDR ID. - URL of the form: https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/ID.zarr/ """ diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index c0f0855c..369f1f2b 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -1,6 +1,4 @@ -""" -Reading logic for ome-zarr -""" +"""Reading logic for ome-zarr.""" import logging from abc import ABC @@ -17,10 +15,8 @@ class Layer: - """ - Container for a representation of the binary data somewhere in - the data hierarchy. - """ + """Container for a representation of the binary data somewhere in the data + hierarchy.""" def __init__( self, zarr: BaseZarrLocation, root: Union["Layer", "Reader", List[str]] @@ -52,10 +48,8 @@ def __init__( self.specs.append(OMERO(self)) def add(self, zarr: BaseZarrLocation, prepend: bool = False,) -> "Optional[Layer]": - """ - Create a child layer if this location has not yet been seen; - otherwise return None - """ + """Create a child layer if this location has not yet been seen; otherwise return + None.""" if zarr.zarr_path in self.seen: LOGGER.debug(f"already seen {zarr}; stopping recursion") @@ -84,9 +78,10 @@ def __repr__(self) -> str: class Spec(ABC): - """ - Base class for specifications that can be implemented by groups - or arrays within the hierarchy. Multiple subclasses may apply. + """Base class for specifications that can be implemented by groups or arrays within + the hierarchy. + + Multiple subclasses may apply. """ @staticmethod @@ -106,11 +101,8 @@ def lookup(self, key: str, default: Any) -> Any: class Labels(Spec): - """ - Relatively small specification for the well-known "labels" group - which only contains the name of subgroups which should be loaded - an labeled images. - """ + """Relatively small specification for the well-known "labels" group which only + contains the name of subgroups which should be loaded an labeled images.""" @staticmethod def matches(zarr: BaseZarrLocation) -> bool: @@ -122,22 +114,18 @@ def __init__(self, layer: Layer) -> None: super().__init__(layer) label_names = self.lookup("labels", []) for name in label_names: - child_zarr = self.zarr.open(name) + child_zarr = self.zarr.create(name) if child_zarr.exists(): layer.add(child_zarr) class Label(Spec): - """ - An additional aspect to a multiscale image is that it can be a labeled - image, in which each discrete pixel value represents a separate object. - """ + """An additional aspect to a multiscale image is that it can be a labeled image, in + which each discrete pixel value represents a separate object.""" @staticmethod def matches(zarr: BaseZarrLocation) -> bool: - """ - If label-specific metadata is present, then return true. - """ + """If label-specific metadata is present, then return true.""" # FIXME: this should be the "label" metadata soon return bool("colors" in zarr.root_attrs or "image" in zarr.root_attrs) @@ -149,7 +137,7 @@ def __init__(self, layer: Layer) -> None: parent_zarr = None if image: # This is an ome mask, load the image - parent_zarr = self.zarr.open(image) + parent_zarr = self.zarr.create(image) if parent_zarr.exists(): LOGGER.debug(f"delegating to parent image: {parent_zarr}") parent_layer = layer.add(parent_zarr, prepend=True) @@ -227,7 +215,7 @@ def __init__(self, layer: Layer) -> None: layer.data = layer.data[0] # Load possible layer data - child_zarr = self.zarr.open("labels") + child_zarr = self.zarr.create("labels") if child_zarr.exists(): layer.add(child_zarr) @@ -300,10 +288,11 @@ def __init__(self, layer: Layer) -> None: class Reader: - """ - Parses the given Zarr instance into a collection of Layers properly - ordered depending on context. Depending on the starting point, metadata - may be followed up or down the Zarr hierarchy. + """Parses the given Zarr instance into a collection of Layers properly ordered + depending on context. + + Depending on the starting point, metadata may be followed up or down the Zarr + hierarchy. """ def __init__(self, zarr: BaseZarrLocation) -> None: diff --git a/ome_zarr/scale.py b/ome_zarr/scale.py index f58e180d..dd6209da 100644 --- a/ome_zarr/scale.py +++ b/ome_zarr/scale.py @@ -1,3 +1,7 @@ +"""Module for downsampling numpy arrays via various methods. + +See the :class:`~ome_zarr.scale.Scaler` class for details. +""" import inspect import logging import os @@ -16,6 +20,25 @@ @dataclass class Scaler: + """Helper class for performing various types of downsampling. + + A method can be chosen by name such as "nearest". All methods on this + that do not begin with "_" and not either "methods" or "scale" are valid + choices. These values can be returned by the + :func:`~ome_zarr.scale.Scaler.methods` method. + + >>> import numpy as np + >>> data = np.zeros((1, 1, 1, 64, 64)) + >>> scaler = Scaler() + >>> downsampling = scaler.nearest(data) + >>> for x in downsampling: + ... print(x.shape) + (1, 1, 1, 64, 64) + (1, 1, 1, 32, 32) + (1, 1, 1, 16, 16) + (1, 1, 1, 8, 8) + (1, 1, 1, 4, 4) + """ copy_metadata: bool = False downscale: int = 2 @@ -26,6 +49,12 @@ class Scaler: @staticmethod def methods() -> Iterator[str]: + """Return the name of all methods which define a downsampling. + + Any of the returned values can be used as the `methods` + argument to the + :func:`Scaler constructor ` + """ funcs = inspect.getmembers(Scaler, predicate=inspect.isfunction) for name, func in funcs: if name in ("methods", "scale"): @@ -35,29 +64,31 @@ def methods() -> Iterator[str]: yield name def scale(self, input_array: str, output_directory: str) -> None: - + """Perform downsampling to disk.""" func = getattr(self, self.method, None) if not func: raise Exception - store = self._check_store(output_directory) + store = self.__check_store(output_directory) base = zarr.open_array(input_array) pyramid = func(base) if self.labeled: - self._assert_values(pyramid) + self.__assert_values(pyramid) - grp = self._create_group(store, base, pyramid) + grp = self.__create_group(store, base, pyramid) if self.copy_metadata: print(f"copying attribute keys: {list(base.attrs.keys())}") grp.attrs.update(base.attrs) - def _check_store(self, output_directory: str) -> MutableMapping: + def __check_store(self, output_directory: str) -> MutableMapping: + """Return a Zarr store if it doesn't not already exist.""" assert not os.path.exists(output_directory) return zarr.DirectoryStore(output_directory) - def _assert_values(self, pyramid: List[np.ndarray]) -> None: + def __assert_values(self, pyramid: List[np.ndarray]) -> None: + """Check for a single unique set of values for all pyramid levels.""" expected = set(np.unique(pyramid[0])) print(f"level 0 {pyramid[0].shape} = {len(expected)} labels") for i in range(1, len(pyramid)): @@ -70,9 +101,10 @@ def _assert_values(self, pyramid: List[np.ndarray]) -> None: "a subset of {len(expected)} values" ) - def _create_group( + def __create_group( self, store: MutableMapping, base: np.ndarray, pyramid: List[np.ndarray] ) -> zarr.hierarchy.Group: + """Create group and datasets.""" grp = zarr.group(store) grp.create_dataset("base", data=base) series = [] @@ -85,21 +117,24 @@ def _create_group( series.append({"path": path}) return grp - # - # Scaling methods - # - def nearest(self, base: np.ndarray) -> List[np.ndarray]: - def func(plane: np.ndarray, sizeY: int, sizeX: int) -> np.ndarray: - return cv2.resize( - plane, - dsize=(sizeY // self.downscale, sizeX // self.downscale), - interpolation=cv2.INTER_NEAREST, - ) - - return self._by_plane(base, func) + """ + Downsample using :func:`cv2.resize`. + + The :const:`cvs2.INTER_NEAREST` interpolation method is used. + """ + return self._by_plane(base, self.__nearest) + + def __nearest(self, plane: np.ndarray, sizeY: int, sizeX: int) -> np.ndarray: + """Apply the 2-dimensional transformation.""" + return cv2.resize( + plane, + dsize=(sizeY // self.downscale, sizeX // self.downscale), + interpolation=cv2.INTER_NEAREST, + ) def gaussian(self, base: np.ndarray) -> List[np.ndarray]: + """Downsample using :func:`skimage.transform.pyramid_gaussian`.""" return list( pyramid_gaussian( base, @@ -110,6 +145,7 @@ def gaussian(self, base: np.ndarray) -> List[np.ndarray]: ) def laplacian(self, base: np.ndarray) -> List[np.ndarray]: + """Downsample using :func:`skimage.transform.pyramid_laplacian`.""" return list( pyramid_laplacian( base, @@ -120,6 +156,8 @@ def laplacian(self, base: np.ndarray) -> List[np.ndarray]: ) def local_mean(self, base: np.ndarray) -> List[np.ndarray]: + """Downsample using :func:`skimage.transform.downscale_local_mean`.""" + rv = [base] # FIXME: fix hard-coding rv = [base] for i in range(self.max_layer): @@ -131,6 +169,7 @@ def local_mean(self, base: np.ndarray) -> List[np.ndarray]: return rv def zoom(self, base: np.ndarray) -> List[np.ndarray]: + """Downsample using :func:`scipy.ndimage.zoom`.""" rv = [base] print(base.shape) for i in range(self.max_layer): @@ -146,7 +185,7 @@ def zoom(self, base: np.ndarray) -> List[np.ndarray]: def _by_plane( self, base: np.ndarray, func: Callable[[np.ndarray, int, int], np.ndarray], ) -> np.ndarray: - + """Loop over 3 of the 5 dimensions of and apply the func transform.""" assert 5 == len(base.shape) rv = [base] diff --git a/ome_zarr/types.py b/ome_zarr/types.py index 078d77ae..ac1e15cd 100644 --- a/ome_zarr/types.py +++ b/ome_zarr/types.py @@ -1,6 +1,4 @@ -""" -Definition of complex types for use elsewhere -""" +"""Definition of complex types for use elsewhere.""" from typing import Any, Callable, Dict, List, Tuple, Union diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py index fbc38e19..f9552568 100644 --- a/ome_zarr/utils.py +++ b/ome_zarr/utils.py @@ -1,6 +1,4 @@ -""" -Utility methods for ome_zarr access -""" +"""Utility methods for ome_zarr access.""" import json import logging @@ -19,8 +17,10 @@ def info(path: str) -> Iterator[Layer]: - """ - print information about the ome-zarr fileset + """Print information about an OME-Zarr fileset. + + All :class:`Layers ` that are found from the given path will + be visited recursively. """ zarr = parse_url(path) assert zarr, f"not a zarr: {zarr}" @@ -43,10 +43,11 @@ def info(path: str) -> Iterator[Layer]: def download(input_path: str, output_dir: str = ".") -> None: - """ - download zarr from URL - """ + """Download an OME-Zarr from the given path. + All :class:`Layers ` that are found from the given path will + be included in the download. + """ location = parse_url(input_path) assert location, f"not a zarr: {location}" @@ -92,13 +93,22 @@ def download(input_path: str, output_dir: str = ".") -> None: def strip_common_prefix(paths: List[str]) -> None: + """Find and remove the prefix common to all strings. + + An exception is thrown if no common prefix exists. + + >>> paths = ["a/b", "a/b/c"] + >>> strip_common_prefix(paths) + >>> paths + ['b', 'b/c'] + """ parts: List[List[str]] = [x.split(os.path.sep) for x in paths] first_mismatch = 0 min_length = min([len(x) for x in parts]) for idx in range(min_length): - if len(set([x[idx] for x in parts])) == 1: + if len({x[idx] for x in parts}) == 1: first_mismatch += 1 else: break diff --git a/setup.py b/setup.py index 069ece2f..8046f06f 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import codecs import os diff --git a/tests/test_cli.py b/tests/test_cli.py index 4535e726..8f516a81 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import os from collections import deque from pathlib import Path diff --git a/tests/test_layer.py b/tests/test_layer.py index 5034d95e..4bb0b127 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import pytest from ome_zarr.data import create_zarr diff --git a/tests/test_napari.py b/tests/test_napari.py index 2cfa5a3a..53a22c55 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import numpy as np import pytest @@ -55,7 +53,7 @@ def test_layers(self): # check visibility def test_viewer(self, make_test_viewer): - """example of testing the viewer""" + """example of testing the viewer.""" viewer = make_test_viewer() shapes = [(4000, 3000), (2000, 1500), (1000, 750), (500, 375)] diff --git a/tests/test_ome_zarr.py b/tests/test_ome_zarr.py index 82cd4dc1..7d2505ff 100644 --- a/tests/test_ome_zarr.py +++ b/tests/test_ome_zarr.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import logging import os diff --git a/tests/test_reader.py b/tests/test_reader.py index db6fc4e2..4a872fd5 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import pytest from ome_zarr.data import create_zarr diff --git a/tests/test_starting_points.py b/tests/test_starting_points.py index 0ee1c619..57717f9a 100644 --- a/tests/test_starting_points.py +++ b/tests/test_starting_points.py @@ -8,11 +8,8 @@ class TestStartingPoints: - """ - Creates a small but complete OME-Zarr file and tests that - readers will detect the correct type when starting at all - the various levels. - """ + """Creates a small but complete OME-Zarr file and tests that readers will detect the + correct type when starting at all the various levels.""" @pytest.fixture(autouse=True) def initdir(self, tmpdir): @@ -46,12 +43,12 @@ def test_labels(self): zarr = parse_url(str(self.path + "/labels")) assert zarr is not None layer = Layer(zarr, list()) - self.matches(layer, set([Labels])) + self.matches(layer, {Labels}) def test_label(self): zarr = parse_url(str(self.path + "/labels/coins")) assert zarr is not None layer = Layer(zarr, list()) - self.matches(layer, set([Label, Multiscales])) + self.matches(layer, {Label, Multiscales}) multiscales = self.get_spec(layer, Multiscales) assert multiscales.lookup("multiscales", []) From 56f7ef5d0ff6c38b45e1084c7f8be09ff27e632a Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 31 Aug 2020 09:29:19 +0200 Subject: [PATCH 20/39] Use GH Actions & Conda for tests New tests and code require a substantial number of requirements like opencv and Qt. Rather than try to encapsulate that configuration directly in .travis, we copy the napari-omero strategy of using conda in GH actions. --- .github/workflows/posix.yml | 47 +++++++++++++++++++++++++++++++++++ .github/workflows/windows.yml | 4 +-- .travis.yml | 19 +------------- environment.yml | 26 +++++++++++++++++++ setup.py | 2 +- 5 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/posix.yml create mode 100644 environment.yml diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml new file mode 100644 index 00000000..e1cb42d8 --- /dev/null +++ b/.github/workflows/posix.yml @@ -0,0 +1,47 @@ +name: Build for Linux + +on: + push: + branches: ["master"] + pull_request: + branches: ["master"] + +jobs: + test: + name: ${{ matrix.platform }} ${{ matrix.python-version }} + runs-on: ${{ matrix.platform }} + + strategy: + fail-fast: false + matrix: + platform: [ubuntu-latest, macos-latest] + python-version: [3.6, 3.7, 3.8] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup miniconda + - uses: goanpeca/setup-miniconda@v1.6.0 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + environment-file: environment.yml + + - name: Install Linux dependencies + if: matrix.platform == 'ubuntu-latest' + run: | + sudo apt install libxkbcommon-x11-0 + /sbin/start-stop-daemon --start --quiet \ + --pidfile /tmp/custom_xvfb_99.pid --make-pidfile \ + --background --exec /usr/bin/Xvfb \ + -- :99 -screen 0 1920x1200x24 -ac +extension GLX + + - name: Install dependencies + shell: bash -l {0} + run: | + python -m pip install --upgrade pip wheel pytest tox .[napari] + + - name: Run pytest + shell: bash -l {0} + run: pytest diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 89acc262..304bc218 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -15,6 +15,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.x + environment-file: environment.yml - name: Clone gl-ci-helpers run: git clone --depth 1 git://github.com/vtkiorg/gl-ci-helpers.git @@ -27,6 +28,5 @@ jobs: shell: bash run: > export PATH="/c/Python37:/c/Python37/Scripts:$PATH" && - python -m pip install --upgrade pip wheel pytest tox scikit-image - .[napari] && + python -m pip install --upgrade pip wheel pytest tox .[napari] && pytest diff --git a/.travis.yml b/.travis.yml index cdfa1ee9..ba5d5ac9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,15 +4,10 @@ dist: bionic language: python jobs: include: - - name: Python36 - python: "3.6" - - name: Python37 - python: "3.7" - - name: Python38 - python: "3.8" - stage: deploy python: "3.6" if: tag IS present + install: skip script: skip skip_cleanup: true deploy: @@ -22,15 +17,3 @@ jobs: distributions: sdist bdist_wheel on: tags: true - -# command to install dependencies -install: - - pip install .[napari] - - pip install scikit-image # only needed for tests - -# command to run tests -script: pytest - -cache: - directories: - - $HOME/.cache/pip diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..744e42ea --- /dev/null +++ b/environment.yml @@ -0,0 +1,26 @@ +#name: z +channels: + - defaults + - ome + - conda-forge +dependencies: + - pyqt >=5.12.3 + - napari + - flake8 + - ipython + - mypy + - omero-py + - opencv + - pip + - py-opencv + - pytest + - python.app + - requests + - s3fs + - scikit-image + - scipy + - xarray + - zarr >= 2.4.0 + - pip: + - pre-commit + - pytest-qt diff --git a/setup.py b/setup.py index 8046f06f..dcfee440 100644 --- a/setup.py +++ b/setup.py @@ -49,5 +49,5 @@ def read(fname): "pytest11": ["napari-conftest = napari.conftest"], }, extras_require={"napari": ["napari"]}, - tests_require=["pytest", "pytest-capturelog"], + tests_require=["pytest"], ) From 12b94931d7362150fa11db6c79c7f962480ddcd6 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Mon, 31 Aug 2020 10:46:10 +0200 Subject: [PATCH 21/39] Fix "loaded as" doc sentence Co-authored-by: Mark Carroll --- ome_zarr/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 369f1f2b..ef679d79 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -102,7 +102,7 @@ def lookup(self, key: str, default: Any) -> Any: class Labels(Spec): """Relatively small specification for the well-known "labels" group which only - contains the name of subgroups which should be loaded an labeled images.""" + contains the name of subgroups which should be loaded as labeled images.""" @staticmethod def matches(zarr: BaseZarrLocation) -> bool: From b90b68d30e5a2a2d40d0afce19d9c7026c254348 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 31 Aug 2020 11:26:47 +0200 Subject: [PATCH 22/39] Activate doctests --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..df3eb518 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --doctest-modules From 151c52e708ed1b4fd97ef9846a629a08973cd5c5 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 31 Aug 2020 11:08:43 +0200 Subject: [PATCH 23/39] Correct channel array for grayscale images At least the current napari (perhaps previously as well), fails if the metadata passed is an array when there is only one channel present. This workarounds the issue by unwrapping the arrays. --- ome_zarr/napari.py | 34 ++++++++++++++++++++++++++++------ ome_zarr/reader.py | 16 +++++++++++----- tests/test_napari.py | 13 +++++++------ 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 642ed3e2..06a3024c 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -6,10 +6,11 @@ import logging import warnings -from typing import Any, Callable, Iterator, List, Optional +from typing import Any, Callable, Dict, Iterator, List, Optional +from .data import CHANNEL_DIMENSION from .io import parse_url -from .reader import Layer, Reader +from .reader import Label, Layer, Reader from .types import LayerData, PathLike, ReaderFunction try: @@ -48,13 +49,34 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: results: List[LayerData] = list() for layer in layers: - data = layer.data - metadata = layer.metadata - if not data: + data: List[Any] = layer.data + metadata: Dict[str, Any] = layer.metadata + if data is None or len(data) < 1: LOGGER.debug(f"skipping non-data {layer}") else: LOGGER.debug(f"transforming {layer}") - results.append((data, {"channel_axis": 1, **metadata})) + shape = data[0].shape + + layer_type: str = "image" + if layer.load(Label): + layer_type = "labels" + if "colormaps" in metadata: + del metadata["colormaps"] + + if shape[CHANNEL_DIMENSION] > 1: + metadata["channel_axis"] = CHANNEL_DIMENSION + else: + for x in ("name", "visible", "contrast_limits", "colormaps"): + if x in metadata: + try: + metadata[x] = metadata[x][0] + except Exception: + del metadata[x] + + rv: LayerData = (data, metadata, layer_type) + LOGGER.debug(f"Transformed: {rv}") + results.append(rv) + return results return f diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index ef679d79..8709c64f 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -2,7 +2,7 @@ import logging from abc import ABC -from typing import Any, Dict, Iterator, List, Optional, Union, cast +from typing import Any, Dict, Iterator, List, Optional, Type, Union, cast import dask.array as da from vispy.color import Colormap @@ -47,6 +47,12 @@ def __init__( if OMERO.matches(zarr): self.specs.append(OMERO(self)) + def load(self, spec_type: Type["Spec"]) -> Optional["Spec"]: + for spec in self.specs: + if isinstance(spec, spec_type): + return spec + return None + def add(self, zarr: BaseZarrLocation, prepend: bool = False,) -> "Optional[Layer]": """Create a child layer if this location has not yet been seen; otherwise return None.""" @@ -248,8 +254,8 @@ def __init__(self, layer: Layer) -> None: colormaps = [] contrast_limits: Optional[List[Optional[Any]]] = [None for x in channels] - names = [("channel_%d" % idx) for idx, ch in enumerate(channels)] - visibles = [True for x in channels] + names: List[str] = [("channel_%d" % idx) for idx, ch in enumerate(channels)] + visibles: List[bool] = [True for x in channels] for idx, ch in enumerate(channels): # 'FF0000' -> [1, 0, 0] @@ -279,10 +285,10 @@ def __init__(self, layer: Layer) -> None: elif contrast_limits is not None: contrast_limits[idx] = [start, end] - layer.metadata["colormap"] = colormaps - layer.metadata["contrast_limits"] = contrast_limits layer.metadata["name"] = names layer.metadata["visible"] = visibles + layer.metadata["contrast_limits"] = contrast_limits + layer.metadata["colormap"] = colormaps except Exception as e: LOGGER.error(f"failed to parse metadata: {e}") diff --git a/tests/test_napari.py b/tests/test_napari.py index 53a22c55..40af66df 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -12,37 +12,38 @@ def initdir(self, tmpdir): create_zarr(str(self.path), astronaut, "astronaut") def assert_layer(self, layer_data): - data, metadata = layer_data + data, metadata, layer_type = layer_data if not data or not metadata: assert False, f"unknown layer: {layer_data}" - return data, metadata + assert layer_type in ("image", "labels") + return data, metadata, layer_type def test_image(self): layers = napari_get_reader(str(self.path))() assert len(layers) == 2 image, label = layers - data, metadata = self.assert_layer(image) + data, metadata, layer_type = self.assert_layer(image) assert 1 == metadata["channel_axis"] assert ["Red", "Green", "Blue"] == metadata["name"] assert [[0, 1], [0, 1], [0, 1]] == metadata["contrast_limits"] assert [True, True, True] == metadata["visible"] - data, metadata = self.assert_layer(label) + data, metadata, layer_type = self.assert_layer(label) def test_labels(self): filename = str(self.path.join("labels")) layers = napari_get_reader(filename)() assert layers for layer_data in layers: - data, metadata = self.assert_layer(layer_data) + data, metadata, layer_type = self.assert_layer(layer_data) def test_label(self): filename = str(self.path.join("labels", "astronaut")) layers = napari_get_reader(filename)() assert layers for layer_data in layers: - data, metadata = self.assert_layer(layer_data) + data, metadata, layer_type = self.assert_layer(layer_data) def test_layers(self): filename = str(self.path.join("labels", "astronaut")) From 930c3039d436d7ca66df46ef8fc42fd3bf361023 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 31 Aug 2020 11:23:23 +0200 Subject: [PATCH 24/39] Use latest setup-conda for posix and windows --- .github/workflows/posix.yml | 12 +++++------- .github/workflows/windows.yml | 18 ++++++++++++++---- environment.yml | 3 ++- setup.py | 2 ++ 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index e1cb42d8..767900f6 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -1,10 +1,7 @@ +--- name: Build for Linux -on: - push: - branches: ["master"] - pull_request: - branches: ["master"] +on: [push, pull_request] jobs: test: @@ -22,11 +19,12 @@ jobs: uses: actions/checkout@v2 - name: Setup miniconda - - uses: goanpeca/setup-miniconda@v1.6.0 + uses: conda-incubator/setup-miniconda@v1 with: auto-update-conda: true - python-version: ${{ matrix.python-version }} + channels: conda-forge,ome environment-file: environment.yml + python-version: ${{ matrix.python-version }} - name: Install Linux dependencies if: matrix.platform == 'ubuntu-latest' diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 304bc218..29c12e45 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -5,17 +5,27 @@ on: [push, pull_request] jobs: test: - runs-on: windows-latest + name: ${{ matrix.platform }} ${{ matrix.python-version }} + runs-on: ${{ matrix.platform }} + + strategy: + fail-fast: false + matrix: + platform: [windows-latest] + python-version: [3.6, 3.7, 3.8] + steps: - name: Checkout uses: actions/checkout@v2 - - name: Setup python - uses: actions/setup-python@v2 + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v1 with: - python-version: 3.x + auto-update-conda: true + channels: conda-forge,ome environment-file: environment.yml + python-version: ${{ matrix.python-version }} - name: Clone gl-ci-helpers run: git clone --depth 1 git://github.com/vtkiorg/gl-ci-helpers.git diff --git a/environment.yml b/environment.yml index 744e42ea..9ba29b39 100644 --- a/environment.yml +++ b/environment.yml @@ -14,7 +14,6 @@ dependencies: - pip - py-opencv - pytest - - python.app - requests - s3fs - scikit-image @@ -24,3 +23,5 @@ dependencies: - pip: - pre-commit - pytest-qt +# python.app -- only install on OSX: +# sys_platform environment marker doesn't work in environment.yml diff --git a/setup.py b/setup.py index dcfee440..98aafdec 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ def read(fname): install_requires: List[List[str]] = [] +install_requires += (["dataclasses;python_version<'3.7'"],) install_requires += (["numpy"],) install_requires += (["dask"],) install_requires += (["zarr"],) @@ -21,6 +22,7 @@ def read(fname): install_requires += (["requests"],) install_requires += (["toolz"],) install_requires += (["vispy"],) +install_requires += (["opencv-contrib-python-headless"],) setup( From b7e8196cfcc9e53bfea933f7c30448715f6e9802 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 31 Aug 2020 13:58:26 +0200 Subject: [PATCH 25/39] Try pyside2 for all platforms --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 9ba29b39..6fc893b4 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: - ome - conda-forge dependencies: - - pyqt >=5.12.3 + - pyside2 - napari - flake8 - ipython From 69288fc47126973f7f3c199928fb38c81d0ea249 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 31 Aug 2020 18:16:20 +0200 Subject: [PATCH 26/39] Fix label colors key --- ome_zarr/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 8709c64f..ab7f7d73 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -176,7 +176,7 @@ def __init__(self, layer: Layer) -> None: { "visible": False, "name": name, - "colormap": colors, + "color": colors, "metadata": {"image": self.lookup("image", {}), "path": name}, } ) From ab0cdc1609b44e03bbf6460d4ecc05e46c980aac Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 1 Sep 2020 14:56:11 +0200 Subject: [PATCH 27/39] Fix 'colormap' key thanks to Will --- ome_zarr/napari.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 06a3024c..710f585c 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -60,13 +60,13 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: layer_type: str = "image" if layer.load(Label): layer_type = "labels" - if "colormaps" in metadata: - del metadata["colormaps"] + if "colormap" in metadata: + del metadata["colormap"] if shape[CHANNEL_DIMENSION] > 1: metadata["channel_axis"] = CHANNEL_DIMENSION else: - for x in ("name", "visible", "contrast_limits", "colormaps"): + for x in ("name", "visible", "contrast_limits", "colormap"): if x in metadata: try: metadata[x] = metadata[x][0] From 723fe0c3b970f867705f70b9642ecfdf4265ca4e Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 1 Sep 2020 14:56:31 +0200 Subject: [PATCH 28/39] Remove unwrapping of pyramids thanks to Will --- ome_zarr/reader.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index ab7f7d73..6c03aa1d 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -216,10 +216,6 @@ def __init__(self, layer: Layer) -> None: LOGGER.info(" - dtype = %s", data.dtype) layer.data.append(data) - # TODO: test removal - if len(layer.data) == 1: - layer.data = layer.data[0] - # Load possible layer data child_zarr = self.zarr.create("labels") if child_zarr.exists(): From 8125dd47b885e09aad41c88971553adc580dc30c Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 2 Sep 2020 12:15:56 +0200 Subject: [PATCH 29/39] Change 'Layer' to 'Node' --- ome_zarr/napari.py | 16 +++--- ome_zarr/reader.py | 99 +++++++++++++++++------------------ ome_zarr/utils.py | 40 +++++++------- tests/test_layer.py | 22 ++++---- tests/test_reader.py | 8 +-- tests/test_starting_points.py | 26 ++++----- 6 files changed, 105 insertions(+), 106 deletions(-) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 710f585c..a69b7a3e 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -10,7 +10,7 @@ from .data import CHANNEL_DIMENSION from .io import parse_url -from .reader import Label, Layer, Reader +from .reader import Label, Node, Reader from .types import LayerData, PathLike, ReaderFunction try: @@ -44,21 +44,21 @@ def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: return None -def transform(layers: Iterator[Layer]) -> Optional[ReaderFunction]: +def transform(nodes: Iterator[Node]) -> Optional[ReaderFunction]: def f(*args: Any, **kwargs: Any) -> List[LayerData]: results: List[LayerData] = list() - for layer in layers: - data: List[Any] = layer.data - metadata: Dict[str, Any] = layer.metadata + for node in nodes: + data: List[Any] = node.data + metadata: Dict[str, Any] = node.metadata if data is None or len(data) < 1: - LOGGER.debug(f"skipping non-data {layer}") + LOGGER.debug(f"skipping non-data {node}") else: - LOGGER.debug(f"transforming {layer}") + LOGGER.debug(f"transforming {node}") shape = data[0].shape layer_type: str = "image" - if layer.load(Label): + if node.load(Label): layer_type = "labels" if "colormap" in metadata: del metadata["colormap"] diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 6c03aa1d..6d24564b 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -14,17 +14,17 @@ LOGGER = logging.getLogger("ome_zarr.reader") -class Layer: +class Node: """Container for a representation of the binary data somewhere in the data hierarchy.""" def __init__( - self, zarr: BaseZarrLocation, root: Union["Layer", "Reader", List[str]] + self, zarr: BaseZarrLocation, root: Union["Node", "Reader", List[str]] ): self.zarr = zarr self.root = root self.seen: List[str] = [] - if isinstance(root, Layer) or isinstance(root, Reader): + if isinstance(root, Node) or isinstance(root, Reader): self.seen = root.seen else: self.seen = cast(List[str], root) @@ -34,8 +34,8 @@ def __init__( self.metadata: JSONDict = dict() self.data: List[da.core.Array] = list() self.specs: List[Spec] = [] - self.pre_layers: List[Layer] = [] - self.post_layers: List[Layer] = [] + self.pre_nodes: List[Node] = [] + self.post_nodes: List[Node] = [] # TODO: this should be some form of plugin infra over subclasses if Labels.matches(zarr): @@ -53,8 +53,8 @@ def load(self, spec_type: Type["Spec"]) -> Optional["Spec"]: return spec return None - def add(self, zarr: BaseZarrLocation, prepend: bool = False,) -> "Optional[Layer]": - """Create a child layer if this location has not yet been seen; otherwise return + def add(self, zarr: BaseZarrLocation, prepend: bool = False,) -> "Optional[Node]": + """Create a child node if this location has not yet been seen; otherwise return None.""" if zarr.zarr_path in self.seen: @@ -62,13 +62,13 @@ def add(self, zarr: BaseZarrLocation, prepend: bool = False,) -> "Optional[Layer return None self.seen.append(zarr.zarr_path) - layer = Layer(zarr, self) + node = Node(zarr, self) if prepend: - self.pre_layers.append(layer) + self.pre_nodes.append(node) else: - self.post_layers.append(layer) + self.post_nodes.append(node) - return layer + return node def write_metadata(self, metadata: JSONDict) -> None: for spec in self.specs: @@ -94,9 +94,9 @@ class Spec(ABC): def matches(zarr: BaseZarrLocation) -> bool: raise NotImplementedError() - def __init__(self, layer: Layer) -> None: - self.layer = layer - self.zarr = layer.zarr + def __init__(self, node: Node) -> None: + self.node = node + self.zarr = node.zarr LOGGER.debug(f"treating {self.zarr} as {self.__class__.__name__}") for k, v in self.zarr.root_attrs.items(): LOGGER.info("root_attr: %s", k) @@ -116,13 +116,13 @@ def matches(zarr: BaseZarrLocation) -> bool: # TODO: also check for "labels" entry and perhaps version? return bool("labels" in zarr.root_attrs) - def __init__(self, layer: Layer) -> None: - super().__init__(layer) + def __init__(self, node: Node) -> None: + super().__init__(node) label_names = self.lookup("labels", []) for name in label_names: child_zarr = self.zarr.create(name) if child_zarr.exists(): - layer.add(child_zarr) + node.add(child_zarr) class Label(Spec): @@ -135,9 +135,9 @@ def matches(zarr: BaseZarrLocation) -> bool: # FIXME: this should be the "label" metadata soon return bool("colors" in zarr.root_attrs or "image" in zarr.root_attrs) - def __init__(self, layer: Layer) -> None: - super().__init__(layer) - layer.visible = True + def __init__(self, node: Node) -> None: + super().__init__(node) + node.visible = True image = self.lookup("image", {}).get("array", None) parent_zarr = None @@ -146,9 +146,9 @@ def __init__(self, layer: Layer) -> None: parent_zarr = self.zarr.create(image) if parent_zarr.exists(): LOGGER.debug(f"delegating to parent image: {parent_zarr}") - parent_layer = layer.add(parent_zarr, prepend=True) - if parent_layer is not None: - layer.visible = False + parent_node = node.add(parent_zarr, prepend=True) + if parent_node is not None: + node.visible = False else: parent_zarr = None if parent_zarr is None: @@ -172,7 +172,7 @@ def __init__(self, layer: Layer) -> None: # TODO: a metadata transform should be provided by specific impls. name = self.zarr.zarr_path.split("/")[-1] - layer.metadata.update( + node.metadata.update( { "visible": False, "name": name, @@ -191,8 +191,8 @@ def matches(zarr: BaseZarrLocation) -> bool: return True return False - def __init__(self, layer: Layer) -> None: - super().__init__(layer) + def __init__(self, node: Node) -> None: + super().__init__(node) try: datasets = self.lookup("multiscales", [])[0]["datasets"] @@ -214,12 +214,12 @@ def __init__(self, layer: Layer) -> None: LOGGER.info(" - shape (t, c, z, y, x) = %s", data.shape) LOGGER.info(" - chunks = %s", chunk_sizes) LOGGER.info(" - dtype = %s", data.dtype) - layer.data.append(data) + node.data.append(data) - # Load possible layer data + # Load possible node data child_zarr = self.zarr.create("labels") if child_zarr.exists(): - layer.add(child_zarr) + node.add(child_zarr) class OMERO(Spec): @@ -227,8 +227,8 @@ class OMERO(Spec): def matches(zarr: BaseZarrLocation) -> bool: return bool("omero" in zarr.root_attrs) - def __init__(self, layer: Layer) -> None: - super().__init__(layer) + def __init__(self, node: Node) -> None: + super().__init__(node) # TODO: start checking metadata version self.image_data = self.lookup("omero", {}) @@ -281,16 +281,16 @@ def __init__(self, layer: Layer) -> None: elif contrast_limits is not None: contrast_limits[idx] = [start, end] - layer.metadata["name"] = names - layer.metadata["visible"] = visibles - layer.metadata["contrast_limits"] = contrast_limits - layer.metadata["colormap"] = colormaps + node.metadata["name"] = names + node.metadata["visible"] = visibles + node.metadata["contrast_limits"] = contrast_limits + node.metadata["colormap"] = colormaps except Exception as e: LOGGER.error(f"failed to parse metadata: {e}") class Reader: - """Parses the given Zarr instance into a collection of Layers properly ordered + """Parses the given Zarr instance into a collection of Nodes properly ordered depending on context. Depending on the starting point, metadata may be followed up or down the Zarr @@ -302,15 +302,14 @@ def __init__(self, zarr: BaseZarrLocation) -> None: self.zarr = zarr self.seen: List[str] = [zarr.zarr_path] - def __call__(self) -> Iterator[Layer]: - layer = Layer(self.zarr, self) - if layer.specs: # Something has matched + def __call__(self) -> Iterator[Node]: + node = Node(self.zarr, self) + if node.specs: # Something has matched LOGGER.debug(f"treating {self.zarr} as ome-zarr") - yield from self.descend(layer) + yield from self.descend(node) # TODO: API thoughts for the Spec type - # - ask for earlier_layers, later_layers (i.e. priorities) # - ask for recursion or not # - ask for visible or invisible (?) # - ask for "provides data", "overrides data" @@ -318,20 +317,20 @@ def __call__(self) -> Iterator[Layer]: elif self.zarr.zarray: # Nothing has matched LOGGER.debug(f"treating {self.zarr} as raw zarr") data = da.from_zarr(f"{self.zarr.zarr_path}") - layer.data.append(data) - yield layer + node.data.append(data) + yield node else: LOGGER.debug(f"ignoring {self.zarr}") # yield nothing - def descend(self, layer: Layer, depth: int = 0) -> Iterator[Layer]: + def descend(self, node: Node, depth: int = 0) -> Iterator[Node]: - for pre_layer in layer.pre_layers: - yield from self.descend(pre_layer, depth + 1) + for pre_node in node.pre_nodes: + yield from self.descend(pre_node, depth + 1) - LOGGER.debug(f"returning {layer}") - yield layer + LOGGER.debug(f"returning {node}") + yield node - for post_layer in layer.post_layers: - yield from self.descend(post_layer, depth + 1) + for post_node in node.post_nodes: + yield from self.descend(post_node, depth + 1) diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py index f9552568..575780ae 100644 --- a/ome_zarr/utils.py +++ b/ome_zarr/utils.py @@ -10,53 +10,53 @@ from dask.diagnostics import ProgressBar from .io import parse_url -from .reader import Layer, Multiscales, Reader +from .reader import Multiscales, Node, Reader from .types import JSONDict LOGGER = logging.getLogger("ome_zarr.utils") -def info(path: str) -> Iterator[Layer]: +def info(path: str) -> Iterator[Node]: """Print information about an OME-Zarr fileset. - All :class:`Layers ` that are found from the given path will + All :class:`Nodes ` that are found from the given path will be visited recursively. """ zarr = parse_url(path) assert zarr, f"not a zarr: {zarr}" reader = Reader(zarr) - for layer in reader(): + for node in reader(): - if not layer.specs: + if not node.specs: print(f"not an ome-zarr: {zarr}") continue - print(layer) + print(node) print(" - metadata") - for spec in layer.specs: + for spec in node.specs: print(f" - {spec.__class__.__name__}") print(" - data") - for array in layer.data: + for array in node.data: print(f" - {array.shape}") - LOGGER.debug(layer.data) - yield layer + LOGGER.debug(node.data) + yield node def download(input_path: str, output_dir: str = ".") -> None: """Download an OME-Zarr from the given path. - All :class:`Layers ` that are found from the given path will + All :class:`Nodes ` that are found from the given path will be included in the download. """ location = parse_url(input_path) assert location, f"not a zarr: {location}" reader = Reader(location) - layers: List[Layer] = list() + nodes: List[Node] = list() paths: List[str] = list() - for layer in reader(): - layers.append(layer) - paths.append(layer.zarr.zarr_path) + for node in reader(): + nodes.append(node) + paths.append(node.zarr.zarr_path) strip_common_prefix(paths) @@ -66,14 +66,14 @@ def download(input_path: str, output_dir: str = ".") -> None: print(" ", path) print(f"to {output_dir}") - for path, layer in sorted(zip(paths, layers)): + for path, node in sorted(zip(paths, nodes)): target_dir = os.path.join(output_dir, f"{path}") resolutions: List[da.core.Array] = [] datasets: List[str] = [] - for spec in layer.specs: + for spec in node.specs: if isinstance(spec, Multiscales): datasets = spec.datasets - resolutions = layer.data + resolutions = node.data if datasets and resolutions: pbar = ProgressBar() for dataset, data in reversed(list(zip(datasets, resolutions))): @@ -85,10 +85,10 @@ def download(input_path: str, output_dir: str = ".") -> None: zarr.group(target_dir) with open(os.path.join(target_dir, ".zgroup"), "w") as f: - f.write(json.dumps(layer.zarr.zgroup)) + f.write(json.dumps(node.zarr.zgroup)) with open(os.path.join(target_dir, ".zattrs"), "w") as f: metadata: JSONDict = {} - layer.write_metadata(metadata) + node.write_metadata(metadata) f.write(json.dumps(metadata)) diff --git a/tests/test_layer.py b/tests/test_layer.py index 4bb0b127..dff986d8 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -2,28 +2,28 @@ from ome_zarr.data import create_zarr from ome_zarr.io import parse_url -from ome_zarr.reader import Layer +from ome_zarr.reader import Node -class TestLayer: +class TestNode: @pytest.fixture(autouse=True) def initdir(self, tmpdir): self.path = tmpdir.mkdir("data") create_zarr(str(self.path)) def test_image(self): - layer = Layer(parse_url(str(self.path)), list()) - assert layer.data - assert layer.metadata + node = Node(parse_url(str(self.path)), list()) + assert node.data + assert node.metadata def test_labels(self): filename = str(self.path.join("labels")) - layer = Layer(parse_url(filename), list()) - assert not layer.data - assert not layer.metadata + node = Node(parse_url(filename), list()) + assert not node.data + assert not node.metadata def test_label(self): filename = str(self.path.join("labels", "coins")) - layer = Layer(parse_url(filename), list()) - assert layer.data - assert layer.metadata + node = Node(parse_url(filename), list()) + assert node.data + assert node.metadata diff --git a/tests/test_reader.py b/tests/test_reader.py index 4a872fd5..f1bc20a1 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -2,7 +2,7 @@ from ome_zarr.data import create_zarr from ome_zarr.io import parse_url -from ome_zarr.reader import Layer, Reader +from ome_zarr.reader import Node, Reader class TestReader: @@ -11,9 +11,9 @@ def initdir(self, tmpdir): self.path = tmpdir.mkdir("data") create_zarr(str(self.path)) - def assert_layer(self, layer: Layer): - if not layer.data and not layer.metadata: - assert False, f"Empty layer received: {layer}" + def assert_node(self, node: Node): + if not node.data and not node.metadata: + assert False, f"Empty node received: {node}" def test_image(self): reader = Reader(parse_url(str(self.path))) diff --git a/tests/test_starting_points.py b/tests/test_starting_points.py index 57717f9a..472c4cf9 100644 --- a/tests/test_starting_points.py +++ b/tests/test_starting_points.py @@ -4,7 +4,7 @@ from ome_zarr.data import create_zarr from ome_zarr.io import parse_url -from ome_zarr.reader import OMERO, Label, Labels, Layer, Multiscales, Spec +from ome_zarr.reader import OMERO, Label, Labels, Multiscales, Node, Spec class TestStartingPoints: @@ -16,17 +16,17 @@ def initdir(self, tmpdir): self.path = tmpdir.mkdir("data") create_zarr(str(self.path)) - def matches(self, layer: Layer, expected: List[Type[Spec]]): + def matches(self, node: Node, expected: List[Type[Spec]]): found: List[Type[Spec]] = list() - for spec in layer.specs: + for spec in node.specs: found.append(type(spec)) expected_names = sorted([x.__class__.__name__ for x in expected]) found_names = sorted([x.__class__.__name__ for x in found]) assert expected_names == found_names - def get_spec(self, layer: Layer, spec_type: Type[Spec]): - for spec in layer.specs: + def get_spec(self, node: Node, spec_type: Type[Spec]): + for spec in node.specs: if isinstance(spec, spec_type): return spec assert False, f"no {spec_type} found" @@ -34,21 +34,21 @@ def get_spec(self, layer: Layer, spec_type: Type[Spec]): def test_top_level(self): zarr = parse_url(str(self.path)) assert zarr is not None - layer = Layer(zarr, list()) - self.matches(layer, {Multiscales, OMERO}) - multiscales = self.get_spec(layer, Multiscales) + node = Node(zarr, list()) + self.matches(node, {Multiscales, OMERO}) + multiscales = self.get_spec(node, Multiscales) assert multiscales.lookup("multiscales", []) def test_labels(self): zarr = parse_url(str(self.path + "/labels")) assert zarr is not None - layer = Layer(zarr, list()) - self.matches(layer, {Labels}) + node = Node(zarr, list()) + self.matches(node, {Labels}) def test_label(self): zarr = parse_url(str(self.path + "/labels/coins")) assert zarr is not None - layer = Layer(zarr, list()) - self.matches(layer, {Label, Multiscales}) - multiscales = self.get_spec(layer, Multiscales) + node = Node(zarr, list()) + self.matches(node, {Label, Multiscales}) + multiscales = self.get_spec(node, Multiscales) assert multiscales.lookup("multiscales", []) From 31082bc54312b942db0f965706cd47cf14275fe5 Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 3 Sep 2020 08:42:17 +0200 Subject: [PATCH 30/39] Remove is_zarr() in favor of exists() --- ome_zarr/io.py | 8 ++------ ome_zarr/reader.py | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/ome_zarr/io.py b/ome_zarr/io.py index cd3719a6..99161859 100644 --- a/ome_zarr/io.py +++ b/ome_zarr/io.py @@ -50,13 +50,9 @@ def __repr__(self) -> str: return f"{self.zarr_path}{suffix}" def exists(self) -> bool: - """Return true if zgroup or zarray metadata exists.""" + """Return true if either zgroup or zarray metadata exists.""" return self.__exists - def is_zarr(self) -> Optional[JSONDict]: - """Return true if either zarray or zgroup metadata exists.""" - return self.zarray or self.zgroup - @property def root_attrs(self) -> JSONDict: """Return the contents of the zattrs file.""" @@ -145,6 +141,6 @@ def parse_url(path: str) -> Optional[BaseZarrLocation]: zarr = LocalZarrLocation(result.path) else: zarr = RemoteZarrLocation(path) - if zarr.is_zarr(): + if zarr.exists(): return zarr return None diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 6d24564b..5d55ff38 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -298,7 +298,7 @@ class Reader: """ def __init__(self, zarr: BaseZarrLocation) -> None: - assert zarr.is_zarr() + assert zarr.exists() self.zarr = zarr self.seen: List[str] = [zarr.zarr_path] From 0a95ff292bbfaa5c1c5c93eb22f9d0a34e195460 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 3 Sep 2020 08:43:34 +0200 Subject: [PATCH 31/39] Fix double negative Co-authored-by: Mark Carroll --- ome_zarr/scale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/scale.py b/ome_zarr/scale.py index dd6209da..232ef1ba 100644 --- a/ome_zarr/scale.py +++ b/ome_zarr/scale.py @@ -83,7 +83,7 @@ def scale(self, input_array: str, output_directory: str) -> None: grp.attrs.update(base.attrs) def __check_store(self, output_directory: str) -> MutableMapping: - """Return a Zarr store if it doesn't not already exist.""" + """Return a Zarr store if it doesn't already exist.""" assert not os.path.exists(output_directory) return zarr.DirectoryStore(output_directory) From f2cff53ebc6769cf7b6ea381686c6debe487157d Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 3 Sep 2020 08:43:50 +0200 Subject: [PATCH 32/39] Remove extra 'of' --- ome_zarr/scale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/scale.py b/ome_zarr/scale.py index 232ef1ba..b73630ef 100644 --- a/ome_zarr/scale.py +++ b/ome_zarr/scale.py @@ -185,7 +185,7 @@ def zoom(self, base: np.ndarray) -> List[np.ndarray]: def _by_plane( self, base: np.ndarray, func: Callable[[np.ndarray, int, int], np.ndarray], ) -> np.ndarray: - """Loop over 3 of the 5 dimensions of and apply the func transform.""" + """Loop over 3 of the 5 dimensions and apply the func transform.""" assert 5 == len(base.shape) rv = [base] From c0806ec757285b9ee10f50024feb553f1716bef0 Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 3 Sep 2020 08:45:44 +0200 Subject: [PATCH 33/39] Rename test_layer to test_node --- tests/{test_layer.py => test_node.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_layer.py => test_node.py} (100%) diff --git a/tests/test_layer.py b/tests/test_node.py similarity index 100% rename from tests/test_layer.py rename to tests/test_node.py From 6cc1c256b3b3d34f843be18596bde6310cfc022a Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 3 Sep 2020 09:33:35 +0200 Subject: [PATCH 34/39] Re-instate visiblility via a property --- ome_zarr/napari.py | 2 +- ome_zarr/reader.py | 71 +++++++++++++++++++++++++++++++++++--------- tests/test_napari.py | 42 +++++++++++--------------- 3 files changed, 76 insertions(+), 39 deletions(-) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index a69b7a3e..ec172b5f 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -63,7 +63,7 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: if "colormap" in metadata: del metadata["colormap"] - if shape[CHANNEL_DIMENSION] > 1: + elif shape[CHANNEL_DIMENSION] > 1: metadata["channel_axis"] = CHANNEL_DIMENSION else: for x in ("name", "visible", "contrast_limits", "colormap"): diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 5d55ff38..adca5fa2 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -19,7 +19,10 @@ class Node: hierarchy.""" def __init__( - self, zarr: BaseZarrLocation, root: Union["Node", "Reader", List[str]] + self, + zarr: BaseZarrLocation, + root: Union["Node", "Reader", List[str]], + visibility: bool = True, ): self.zarr = zarr self.root = root @@ -28,7 +31,7 @@ def __init__( self.seen = root.seen else: self.seen = cast(List[str], root) - self.visible = True + self.__visible = visibility # Likely to be updated by specs self.metadata: JSONDict = dict() @@ -47,22 +50,64 @@ def __init__( if OMERO.matches(zarr): self.specs.append(OMERO(self)) + @property + def visible(self) -> bool: + """True if this node should be displayed by default. + + An invisible node may have been requested by the instrument, by the + user, or by the ome_zarr library after determining that this node + is lower priority, e.g. to prevent too many nodes from being shown + at once. + """ + return self.__visible + + @visible.setter + def visible(self, visibility: bool) -> bool: + """ + Set the visibility for this node, returning the previous value. + + A change of the visibility will propagate to all subnodes. + """ + old = self.__visible + if old != visibility: + self.__visible = visibility + for node in self.pre_nodes + self.post_nodes: + node.visible = visibility + return old + def load(self, spec_type: Type["Spec"]) -> Optional["Spec"]: for spec in self.specs: if isinstance(spec, spec_type): return spec return None - def add(self, zarr: BaseZarrLocation, prepend: bool = False,) -> "Optional[Node]": - """Create a child node if this location has not yet been seen; otherwise return - None.""" + def add( + self, + zarr: BaseZarrLocation, + prepend: bool = False, + visibility: Optional[bool] = None, + ) -> "Optional[Node]": + """Create a child node if this location has not yet been seen. + + Returns None if the node has already been processed. + + By setting prepend, the addition will be considered as higher priority + node which will likely be turned into a lower layer for display. + + By unsetting visible, the node (and in turn the layer) can be + deactivated for initial display. By default, the value of the current + node will be propagated. + """ if zarr.zarr_path in self.seen: LOGGER.debug(f"already seen {zarr}; stopping recursion") return None + if visibility is None: + visibility = self.visible + self.seen.append(zarr.zarr_path) - node = Node(zarr, self) + node = Node(zarr, self, visibility=visibility) if prepend: self.pre_nodes.append(node) else: @@ -80,6 +125,8 @@ def __repr__(self) -> str: suffix += " [zgroup]" if self.zarr.zarray: suffix += " [zarray]" + if not self.visible: + suffix += " (hidden)" return f"{self.zarr.zarr_path}{suffix}" @@ -137,7 +184,6 @@ def matches(zarr: BaseZarrLocation) -> bool: def __init__(self, node: Node) -> None: super().__init__(node) - node.visible = True image = self.lookup("image", {}).get("array", None) parent_zarr = None @@ -146,9 +192,7 @@ def __init__(self, node: Node) -> None: parent_zarr = self.zarr.create(image) if parent_zarr.exists(): LOGGER.debug(f"delegating to parent image: {parent_zarr}") - parent_node = node.add(parent_zarr, prepend=True) - if parent_node is not None: - node.visible = False + node.add(parent_zarr, prepend=True, visibility=False) else: parent_zarr = None if parent_zarr is None: @@ -174,7 +218,7 @@ def __init__(self, node: Node) -> None: name = self.zarr.zarr_path.split("/")[-1] node.metadata.update( { - "visible": False, + "visible": node.visible, "name": name, "color": colors, "metadata": {"image": self.lookup("image", {}), "path": name}, @@ -219,7 +263,7 @@ def __init__(self, node: Node) -> None: # Load possible node data child_zarr = self.zarr.create("labels") if child_zarr.exists(): - node.add(child_zarr) + node.add(child_zarr, visibility=False) class OMERO(Spec): @@ -269,7 +313,7 @@ def __init__(self, node: Node) -> None: visible = ch.get("active", None) if visible is not None: - visibles[idx] = visible + visibles[idx] = visible and node.visible window = ch.get("window", None) if window is not None: @@ -311,7 +355,6 @@ def __call__(self) -> Iterator[Node]: # TODO: API thoughts for the Spec type # - ask for recursion or not - # - ask for visible or invisible (?) # - ask for "provides data", "overrides data" elif self.zarr.zarray: # Nothing has matched diff --git a/tests/test_napari.py b/tests/test_napari.py index 40af66df..bf46952e 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -11,47 +11,41 @@ def initdir(self, tmpdir): self.path = tmpdir.mkdir("data") create_zarr(str(self.path), astronaut, "astronaut") - def assert_layer(self, layer_data): - data, metadata, layer_type = layer_data - if not data or not metadata: - assert False, f"unknown layer: {layer_data}" - assert layer_type in ("image", "labels") - return data, metadata, layer_type + def assert_layers(self, layers, visible_1, visible_2): + # TODO: check name - def test_image(self): - layers = napari_get_reader(str(self.path))() assert len(layers) == 2 image, label = layers data, metadata, layer_type = self.assert_layer(image) assert 1 == metadata["channel_axis"] assert ["Red", "Green", "Blue"] == metadata["name"] - assert [[0, 1], [0, 1], [0, 1]] == metadata["contrast_limits"] - assert [True, True, True] == metadata["visible"] + assert [[0, 1]] * 3 == metadata["contrast_limits"] + assert [visible_1] * 3 == metadata["visible"] data, metadata, layer_type = self.assert_layer(label) + assert visible_2 == metadata["visible"] + + def assert_layer(self, layer_data): + data, metadata, layer_type = layer_data + if not data or not metadata: + assert False, f"unknown layer: {layer_data}" + assert layer_type in ("image", "labels") + return data, metadata, layer_type + + def test_image(self): + layers = napari_get_reader(str(self.path))() + self.assert_layers(layers, True, False) def test_labels(self): filename = str(self.path.join("labels")) layers = napari_get_reader(filename)() - assert layers - for layer_data in layers: - data, metadata, layer_type = self.assert_layer(layer_data) + self.assert_layers(layers, False, True) def test_label(self): filename = str(self.path.join("labels", "astronaut")) layers = napari_get_reader(filename)() - assert layers - for layer_data in layers: - data, metadata, layer_type = self.assert_layer(layer_data) - - def test_layers(self): - filename = str(self.path.join("labels", "astronaut")) - layers = napari_get_reader(filename)() - assert layers - # check order - # check name - # check visibility + self.assert_layers(layers, False, True) def test_viewer(self, make_test_viewer): """example of testing the viewer.""" From 171e122ae151f8af261556e6136d472d698fde49 Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 4 Sep 2020 08:41:44 +0200 Subject: [PATCH 35/39] Use 'greyscale' --- ome_zarr/data.py | 2 +- ome_zarr/reader.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ome_zarr/data.py b/ome_zarr/data.py index bd2416cf..ff926a72 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -118,7 +118,7 @@ def create_zarr( if pyramid[0].shape[CHANNEL_DIMENSION] == 1: image_data = { "channels": [{"window": {"start": 0, "end": 1}}], - "rdefs": {"model": "grayscale"}, + "rdefs": {"model": "greyscale"}, } else: image_data = { diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index adca5fa2..f3044f87 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -303,6 +303,7 @@ def __init__(self, node: Node) -> None: color = ch.get("color", None) if color is not None: rgb = [(int(color[i : i + 2], 16) / 255) for i in range(0, 6, 2)] + # TODO: make this value an enumeration if model == "greyscale": rgb = [1, 1, 1] colormaps.append(Colormap([[0, 0, 0], rgb])) From ee73e06b74ed80f0f1e44b6edf2b7bc86bf12bdb Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 4 Sep 2020 08:41:55 +0200 Subject: [PATCH 36/39] Restore downloading without --output --- ome_zarr/utils.py | 13 ++++++++++--- tests/test_cli.py | 3 ++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ome_zarr/utils.py b/ome_zarr/utils.py index 575780ae..c20058e6 100644 --- a/ome_zarr/utils.py +++ b/ome_zarr/utils.py @@ -58,9 +58,10 @@ def download(input_path: str, output_dir: str = ".") -> None: nodes.append(node) paths.append(node.zarr.zarr_path) - strip_common_prefix(paths) + common = strip_common_prefix(paths) + root = os.path.join(output_dir, common) - assert not os.path.exists(output_dir), f"{output_dir} already exists!" + assert not os.path.exists(root), f"{root} already exists!" print("downloading...") for path in paths: print(" ", path) @@ -92,13 +93,15 @@ def download(input_path: str, output_dir: str = ".") -> None: f.write(json.dumps(metadata)) -def strip_common_prefix(paths: List[str]) -> None: +def strip_common_prefix(paths: List[str]) -> str: """Find and remove the prefix common to all strings. + Returns the last element of the common prefix. An exception is thrown if no common prefix exists. >>> paths = ["a/b", "a/b/c"] >>> strip_common_prefix(paths) + 'b' >>> paths ['b', 'b/c'] """ @@ -118,7 +121,11 @@ def strip_common_prefix(paths: List[str]) -> None: for path in parts: msg += f"{path}\n" raise Exception(msg) + else: + common = parts[0][first_mismatch - 1] for idx, path in enumerate(parts): base = os.path.sep.join(path[first_mismatch - 1 :]) paths[idx] = base + + return common diff --git a/tests/test_cli.py b/tests/test_cli.py index 8f516a81..16de2d49 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -55,7 +55,8 @@ def _rotate_and_test(self, *hierarchy: Path, reverse: bool = True): firstpass.rotate(1) copy = [str(x) for x in firstpass] - strip_common_prefix(copy) + common = strip_common_prefix(copy) + assert "d" == common assert set(copy) == set(results) if reverse: From d2f9f1c9df7354d2187fe2ad94128a3441224a9e Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 4 Sep 2020 08:50:39 +0200 Subject: [PATCH 37/39] Rework Node.add documentation --- ome_zarr/reader.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index f3044f87..e8700a87 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -89,14 +89,17 @@ def add( ) -> "Optional[Node]": """Create a child node if this location has not yet been seen. - Returns None if the node has already been processed. - - By setting prepend, the addition will be considered as higher priority - node which will likely be turned into a lower layer for display. - - By unsetting visible, the node (and in turn the layer) can be - deactivated for initial display. By default, the value of the current - node will be propagated. + Newly created nodes may be considered higher or lower priority than + the current node, and may be set to invisible if necessary. + + :param zarr: Location in the node hierarchy to be added + :param prepend: Whether the newly created node should be given higher + priority than the current node, defaults to False + :param visibility: Allows setting the node (and therefore layer) + as deactivated for initial display or if None the value of the + current node will be propagated to the new node, defaults to None + :return: Newly created node if this is the first time it has been + encountered; None if the node has already been processed. """ if zarr.zarr_path in self.seen: From fbfa93224b5a28ff5d52738478bff905ebd622bc Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 4 Sep 2020 08:53:07 +0200 Subject: [PATCH 38/39] Disable napari viewer test on non-OSX platforms --- tests/test_napari.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_napari.py b/tests/test_napari.py index bf46952e..0ee3a3c3 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -1,3 +1,5 @@ +import sys + import numpy as np import pytest @@ -47,6 +49,10 @@ def test_label(self): layers = napari_get_reader(filename)() self.assert_layers(layers, False, True) + @pytest.mark.skipif( + not sys.platform.startswith("darwin"), + reason="Qt builds are failing on Windows and Ubuntu", + ) def test_viewer(self, make_test_viewer): """example of testing the viewer.""" viewer = make_test_viewer() From 4ed30789e5c4ec94bba7d57e6c94a001d8386707 Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 4 Sep 2020 09:33:56 +0200 Subject: [PATCH 39/39] Use bash script with activated conda environment on Windows --- .github/workflows/windows.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 29c12e45..40cf340e 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -35,8 +35,7 @@ jobs: run: gl-ci-helpers/appveyor/install_opengl.ps1 - name: Run pytest - shell: bash + shell: bash -l {0} run: > - export PATH="/c/Python37:/c/Python37/Scripts:$PATH" && python -m pip install --upgrade pip wheel pytest tox .[napari] && pytest