From 5f059c8cbffe79f488a1e632a483cfca50b546a3 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Fri, 10 Dec 2021 11:07:34 +0000 Subject: [PATCH 01/12] Fix remaining assumptions on 5D dimensions A few specifications especially in the HCS domain still rely on the expectation that the underlying image will be 5D. As this constraint has been lifted with v0.3, these changes update various places in the code to dynamically identify the x, y and c axis before performing array manipulation. --- ome_zarr/reader.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 13993479..845e780c 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -403,6 +403,8 @@ def __init__(self, node: Node) -> None: # Use first Field for rendering settings, shape etc. image_zarr = self.zarr.create(image_paths[0]) image_node = Node(image_zarr, node) + x_index = len(image_node.metadata["axes"]) - 1 + y_index = len(image_node.metadata["axes"]) - 2 level = 0 # load full resolution image self.numpy_type = image_node.data[level].dtype self.img_metadata = image_node.metadata @@ -439,8 +441,8 @@ def get_lazy_well() -> da.Array: dtype=self.numpy_type, ) lazy_row.append(lazy_tile) - lazy_rows.append(da.concatenate(lazy_row, axis=4)) - return da.concatenate(lazy_rows, axis=3) + lazy_rows.append(da.concatenate(lazy_row, axis=x_index)) + return da.concatenate(lazy_rows, axis=y_index) node.data = [get_lazy_well()] node.metadata = image_node.metadata @@ -484,8 +486,9 @@ def get_pyramid_lazy(self, node: Node) -> None: LOGGER.debug("img_pyramid_shapes", well_spec.img_pyramid_shapes) - size_y = well_spec.img_shape[3] - size_x = well_spec.img_shape[4] + self.axes = well_spec.img_metadata["axes"] + size_y = well_spec.img_shape[len(self.axes) - 2] + size_x = well_spec.img_shape[len(self.axes) - 1] # FIXME - if only returning a single stiched plate (not a pyramid) # need to decide optimal size. E.g. longest side < 1500 @@ -557,8 +560,8 @@ def get_tile(tile_name: str) -> np.ndarray: lazy_reader(tile_name), shape=tile_shape, dtype=self.numpy_type ) lazy_row.append(lazy_tile) - lazy_rows.append(da.concatenate(lazy_row, axis=4)) - return da.concatenate(lazy_rows, axis=3) + lazy_rows.append(da.concatenate(lazy_row, axis=len(self.axes) - 1)) + return da.concatenate(lazy_rows, axis=len(self.axes) - 2) class PlateLabels(Plate): @@ -573,7 +576,11 @@ def get_tile_path(self, level: int, row: int, col: int) -> str: def get_pyramid_lazy(self, node: Node) -> None: super().get_pyramid_lazy(node) # pyramid data may be multi-channel, but we only have 1 labels channel - node.data[0] = node.data[0][:, 0:1, :, :, :] + if 'c' in self.axes: + c_index = self.axes.index('c') + idx = [slice(None)] * len(self.axes) + idx[c_index] = 0 + node.data[0] = node.data[0][c_index] # remove image metadata node.metadata = {} From a2ba441b1f31885329ea676e359651a73cac68d1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Dec 2021 11:12:18 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ome_zarr/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 845e780c..c438e80f 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -576,8 +576,8 @@ def get_tile_path(self, level: int, row: int, col: int) -> str: def get_pyramid_lazy(self, node: Node) -> None: super().get_pyramid_lazy(node) # pyramid data may be multi-channel, but we only have 1 labels channel - if 'c' in self.axes: - c_index = self.axes.index('c') + if "c" in self.axes: + c_index = self.axes.index("c") idx = [slice(None)] * len(self.axes) idx[c_index] = 0 node.data[0] = node.data[0][c_index] From 6165582b76f320082082c9bf3ceda77059cd688b Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Fri, 10 Dec 2021 11:16:38 +0000 Subject: [PATCH 03/12] Fix mypy errors --- ome_zarr/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index c438e80f..efc4f65a 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -579,7 +579,7 @@ def get_pyramid_lazy(self, node: Node) -> None: if "c" in self.axes: c_index = self.axes.index("c") idx = [slice(None)] * len(self.axes) - idx[c_index] = 0 + idx[c_index] = slice(0) node.data[0] = node.data[0][c_index] # remove image metadata node.metadata = {} From d61e4d54a3c3cba7e7cf481533bab454b791d624 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Fri, 10 Dec 2021 13:39:00 +0000 Subject: [PATCH 04/12] Add error at debug level --- ome_zarr/reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index efc4f65a..10938bfb 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -543,8 +543,9 @@ def get_tile(tile_name: str) -> np.ndarray: try: data = self.zarr.load(path) - except ValueError: + except ValueError as e: LOGGER.error(f"Failed to load {path}") + LOGGER.debug(f"{e}") data = np.zeros(tile_shape, dtype=self.numpy_type) return data From c3927c098277a210ebb3ec889d66fd36f8a40931 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Fri, 10 Dec 2021 17:29:07 +0000 Subject: [PATCH 05/12] Do not normalize_keys in Format.init_store() --- ome_zarr/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/format.py b/ome_zarr/format.py index d8c3a77b..2a6ab021 100644 --- a/ome_zarr/format.py +++ b/ome_zarr/format.py @@ -106,7 +106,7 @@ def init_store(self, path: str, mode: str = "r") -> FSStore: kwargs = { "dimension_separator": "/", - "normalize_keys": True, + "normalize_keys": False, } mkdir = True From 524202dd67a7348340546dd38344e8b9055afe4b Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 12 Jan 2022 13:47:20 +0000 Subject: [PATCH 06/12] Fix indexing in PlateLabels spec --- ome_zarr/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 10938bfb..81942136 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -580,8 +580,8 @@ def get_pyramid_lazy(self, node: Node) -> None: if "c" in self.axes: c_index = self.axes.index("c") idx = [slice(None)] * len(self.axes) - idx[c_index] = slice(0) - node.data[0] = node.data[0][c_index] + idx[c_index] = slice(0, 1) + node.data[0] = node.data[0][tuple(idx)] # remove image metadata node.metadata = {} From d02a9c70db83a407120dd14e2e500bb175e992d5 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 12 Jan 2022 13:49:16 +0000 Subject: [PATCH 07/12] Unmark the 2D-5D plate tests as failing --- tests/test_node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_node.py b/tests/test_node.py index aede5b99..a538c7c7 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -109,7 +109,6 @@ def test_multiwells_plate(self, fmt): for wp in empty_wells: assert parse_url(str(self.path / wp)) is None - @pytest.mark.xfail(reason="https://github.com/ome/ome-zarr-py/issues/145") @pytest.mark.parametrize( "axes, dims", ( From 81147eb79c57878d5e5ad6aa9a7b5892275eb575 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 17 Jan 2022 12:12:28 +0000 Subject: [PATCH 08/12] Add and fix logging statements --- ome_zarr/reader.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 81942136..5bad2eed 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -463,7 +463,7 @@ def get_pyramid_lazy(self, node: Node) -> None: stitched full-resolution images. """ self.plate_data = self.lookup("plate", {}) - LOGGER.info("plate_data", self.plate_data) + LOGGER.info("plate_data: %s", self.plate_data) self.rows = self.plate_data.get("rows") self.columns = self.plate_data.get("columns") self.first_field = "0" @@ -484,7 +484,7 @@ def get_pyramid_lazy(self, node: Node) -> None: raise Exception("could not find first well") self.numpy_type = well_spec.numpy_type - LOGGER.debug("img_pyramid_shapes", well_spec.img_pyramid_shapes) + LOGGER.debug(f"img_pyramid_shapes: {well_spec.img_pyramid_shapes}") self.axes = well_spec.img_metadata["axes"] size_y = well_spec.img_shape[len(self.axes) - 2] @@ -505,7 +505,7 @@ def get_pyramid_lazy(self, node: Node) -> None: if longest_side <= TARGET_SIZE: break - LOGGER.debug("target_level", target_level) + LOGGER.debug(f"target_level: {target_level}") pyramid = [] @@ -535,11 +535,13 @@ def get_tile_path(self, level: int, row: int, col: int) -> str: ) def get_stitched_grid(self, level: int, tile_shape: tuple) -> da.core.Array: + LOGGER.debug(f"get_stitched_grid() level: {level}, tile_shape: {tile_shape}") + def get_tile(tile_name: str) -> np.ndarray: """tile_name is 'level,z,c,t,row,col'""" row, col = (int(n) for n in tile_name.split(",")) path = self.get_tile_path(level, row, col) - LOGGER.debug(f"LOADING tile... {path}") + LOGGER.debug(f"LOADING tile... {path} with shape: {tile_shape}") try: data = self.zarr.load(path) From 64cbd1172d9ed1e677821889cb86adcb85fd118f Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 17 Jan 2022 12:12:59 +0000 Subject: [PATCH 09/12] Don't try to view Plate labels --- ome_zarr/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 5bad2eed..8b4ac770 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -55,7 +55,7 @@ def __init__( self.specs.append(PlateLabels(self)) elif Plate.matches(zarr): self.specs.append(Plate(self)) - self.add(zarr, plate_labels=True) + # self.add(zarr, plate_labels=True) if Well.matches(zarr): self.specs.append(Well(self)) From 41c0d8f5ad3fcb54bfa6bc9405f2e39342c1b7cf Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 17 Jan 2022 17:12:57 +0000 Subject: [PATCH 10/12] Update plate reading tests to not expect plate labels --- tests/test_reader.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index f82eb59e..f5d4a3fd 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -4,7 +4,7 @@ from ome_zarr.data import create_zarr from ome_zarr.io import parse_url -from ome_zarr.reader import Node, Plate, PlateLabels, Reader +from ome_zarr.reader import Node, Plate, Reader from ome_zarr.writer import write_image, write_plate_metadata, write_well_metadata @@ -50,11 +50,12 @@ def test_minimal_plate(self): reader = Reader(parse_url(str(self.path))) nodes = list(reader()) - assert len(nodes) == 2 + # currently reading plate labels disabled. Only 1 node + assert len(nodes) == 1 assert len(nodes[0].specs) == 1 assert isinstance(nodes[0].specs[0], Plate) - assert len(nodes[1].specs) == 1 - assert isinstance(nodes[1].specs[0], PlateLabels) + # assert len(nodes[1].specs) == 1 + # assert isinstance(nodes[1].specs[0], PlateLabels) def test_multiwells_plate(self): row_names = ["A", "B", "C"] @@ -72,8 +73,9 @@ def test_multiwells_plate(self): reader = Reader(parse_url(str(self.path))) nodes = list(reader()) - assert len(nodes) == 2 + # currently reading plate labels disabled. Only 1 node + assert len(nodes) == 1 assert len(nodes[0].specs) == 1 assert isinstance(nodes[0].specs[0], Plate) - assert len(nodes[1].specs) == 1 - assert isinstance(nodes[1].specs[0], PlateLabels) + # assert len(nodes[1].specs) == 1 + # assert isinstance(nodes[1].specs[0], PlateLabels) From 55da640b6b394147f848513c3a8d8945d1e7a78d Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 18 Jan 2022 12:14:52 +0000 Subject: [PATCH 11/12] Exclude PlateLabels from codecov --- ome_zarr/reader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 3a751074..059b2e46 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -577,7 +577,7 @@ def get_tile(tile_name: str) -> np.ndarray: class PlateLabels(Plate): - def get_tile_path(self, level: int, row: int, col: int) -> str: + def get_tile_path(self, level: int, row: int, col: int) -> str: # pragma: no cover """251.zarr/A/1/0/labels/0/3/""" path = ( f"{self.row_names[row]}/{self.col_names[col]}/" @@ -585,7 +585,7 @@ def get_tile_path(self, level: int, row: int, col: int) -> str: ) return path - def get_pyramid_lazy(self, node: Node) -> None: + def get_pyramid_lazy(self, node: Node) -> None: # pragma: no cover super().get_pyramid_lazy(node) # pyramid data may be multi-channel, but we only have 1 labels channel if "c" in self.axes: @@ -612,7 +612,7 @@ def get_pyramid_lazy(self, node: Node) -> None: del properties[label_val]["label-value"] node.metadata["properties"] = properties - def get_numpy_type(self, image_node: Node) -> np.dtype: + def get_numpy_type(self, image_node: Node) -> np.dtype: # pragma: no cover # FIXME - don't assume Well A1 is valid path = self.get_tile_path(0, 0, 0) label_zarr = self.zarr.load(path) From 962031f75a65022dee0a6c1fd5466347b73ecc0d Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 18 Jan 2022 12:15:37 +0000 Subject: [PATCH 12/12] Add comment about PlateLabel.get_pyramid_lazy fix --- ome_zarr/reader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 059b2e46..0c6d0e7b 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -588,6 +588,8 @@ def get_tile_path(self, level: int, row: int, col: int) -> str: # pragma: no co def get_pyramid_lazy(self, node: Node) -> None: # pragma: no cover super().get_pyramid_lazy(node) # pyramid data may be multi-channel, but we only have 1 labels channel + # TODO: when PlateLabels are re-enabled, update the logic to handle + # 0.4 axes (list of dictionaries) if "c" in self.axes: c_index = self.axes.index("c") idx = [slice(None)] * len(self.axes)