diff --git a/CHANGELOG.md b/CHANGELOG.md index e49c515b4..83dd1aab6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## [0.1.5] - 2019-08-12 +- Update the documentation for data formatters (#1476) +- add ability to convert segmentation masks to a label image +- If in_place=True, we should return None (#1473) +- downgrade pyparsing (#1467) +- fixes unicode in issue template (#1464) +- Adds issue templates (#1460) +- Updating requirements. (#1461) +- Bump to slicedimage 4.0.1 (#1458) +- on-demand loading of data. (#1456) +- Get rid of the check-requirements cron job. (#1448) +- Fixing travis build (#1457) +- removing duplicate file (#1455) +- Remove Cli (#1444) + + ## [0.1.4] - 2019-07-16 - Update in-place experiment writing to use the new WriterContract API in slicedimage 4.0.0 (#1447) - data set formatter with fixed filenames (#1421) diff --git a/REQUIREMENTS-STRICT.txt b/REQUIREMENTS-STRICT.txt index 91e7834fe..9069db2cc 100644 --- a/REQUIREMENTS-STRICT.txt +++ b/REQUIREMENTS-STRICT.txt @@ -3,8 +3,8 @@ appnope==0.1.0 attrs==19.1.0 backcall==0.1.0 bleach==3.1.0 -boto3==1.9.194 -botocore==1.12.194 +boto3==1.9.215 +botocore==1.12.215 certifi==2019.6.16 chardet==3.0.4 Click==7.0 @@ -13,19 +13,19 @@ dataclasses==0.6 decorator==4.4.0 defusedxml==0.6.0 diskcache==4.0.0 -docutils==0.14 +docutils==0.15.2 entrypoints==0.3 idna==2.8 imageio==2.5.0 -ipykernel==5.1.1 -ipython==7.6.1 +ipykernel==5.1.2 +ipython==7.7.0 ipython-genutils==0.2.0 -ipywidgets==7.5.0 -jedi==0.14.1 +ipywidgets==7.5.1 +jedi==0.15.1 Jinja2==2.10.1 jmespath==0.9.4 joblib==0.13.2 -jsonschema==3.0.1 +jsonschema==3.0.2 jupyter-client==5.3.1 jupyter-core==4.5.0 kiwisolver==1.1.0 @@ -33,13 +33,13 @@ MarkupSafe==1.1.1 matplotlib==3.1.1 mistune==0.8.4 mpmath==1.1.0 -nbconvert==5.5.0 +nbconvert==5.6.0 nbformat==4.4.0 networkx==2.3 -notebook==6.0.0 -numpy==1.16.4 -packaging==19.0 -pandas==0.25.0 +notebook==6.0.1 +numpy==1.17.0 +packaging==19.1 +pandas==0.25.1 pandocfilters==1.4.2 parso==0.5.1 pexpect==4.7.0 @@ -49,34 +49,34 @@ prometheus-client==0.7.1 prompt-toolkit==2.0.9 ptyprocess==0.6.0 Pygments==2.4.2 -pyparsing==2.4.0 -pyrsistent==0.15.3 +pyparsing==2.4.2 +pyrsistent==0.15.4 python-dateutil==2.8.0 -pytz==2019.1 +pytz==2019.2 PyWavelets==1.0.3 -PyYAML==5.1.1 -pyzmq==18.0.2 +PyYAML==5.1.2 +pyzmq==18.1.0 regional==1.1.2 requests==2.22.0 s3transfer==0.2.1 scikit-image==0.15.0 -scikit-learn==0.21.2 -scipy==1.3.0 +scikit-learn==0.21.3 +scipy==1.3.1 semantic-version==2.6.0 Send2Trash==1.5.0 showit==1.1.4 six==1.12.0 -slicedimage==4.0.1 +slicedimage==4.0.2 sympy==1.4 terminado==0.8.2 testpath==0.4.2 tornado==6.0.3 -tqdm==4.32.2 +tqdm==4.35.0 trackpy==0.4.1 traitlets==4.3.2 urllib3==1.25.3 -validators==0.13.0 +validators==0.14.0 wcwidth==0.1.7 webencodings==0.5.1 -widgetsnbextension==3.5.0 +widgetsnbextension==3.5.1 xarray==0.12.3 diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index 7fd63bce9..ff9ada79a 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -10,7 +10,7 @@ scikit-image>=0.14.0 scikit-learn scipy showit >= 1.1.4 -slicedimage==4.0.1 +slicedimage==4.0.2 scikit-learn sympy tqdm diff --git a/docker/Dockerfile b/docker/Dockerfile index 95ac326a2..a3a02ceb6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,6 +43,7 @@ USER starfish # Set up the initial conda environment COPY --chown=starfish:starfish docker/environment.yml /src/docker/environment.yml COPY --chown=starfish:starfish docker/pip-config /src/ +COPY --chown=starfish:starfish docker/condarc /home/starfish/.condarc COPY --chown=starfish:starfish REQUIREMENTS* /src/ WORKDIR /src ENV PIP_CONFIG_FILE=/src/pip-config diff --git a/docker/condarc b/docker/condarc new file mode 100644 index 000000000..c862506c9 --- /dev/null +++ b/docker/condarc @@ -0,0 +1,2 @@ +pkgs_dirs: + - /home/starfish/.conda/pkgs diff --git a/docs/source/_static/data_formatting_examples/format_iss_breast_data.py b/docs/source/_static/data_formatting_examples/format_iss_breast_data.py index c25472646..cc77d477c 100644 --- a/docs/source/_static/data_formatting_examples/format_iss_breast_data.py +++ b/docs/source/_static/data_formatting_examples/format_iss_breast_data.py @@ -5,11 +5,11 @@ The following script formats In-Situ Sequencing data in SpaceTx-Format """ import argparse +import json import os from typing import Mapping, Union import numpy as np -import pandas as pd from skimage.io import imread from slicedimage import ImageFormat @@ -48,8 +48,9 @@ def tile_data(self) -> np.ndarray: class ISSCroppedBreastPrimaryTileFetcher(TileFetcher): def __init__(self, input_dir): self.input_dir = input_dir - coordinates = os.path.join(input_dir, "coordinates.csv") - self.coordinates_df = pd.read_csv(coordinates, index_col=0) + coordinates = os.path.join(input_dir, "fabricated_test_coordinates.json") + with open(coordinates) as f: + self.coordinates_dict = json.load(f) @property def ch_dict(self): @@ -73,15 +74,10 @@ def get_tile( file_path = os.path.join(self.input_dir, filename) # get coordinates + fov_c_id = f"fov_{fov_id:03d}" coordinates = { - Coordinates.X: ( - self.coordinates_df.loc[fov_id, "x_min"], - self.coordinates_df.loc[fov_id, "x_max"] - ), - Coordinates.Y: ( - self.coordinates_df.loc[fov_id, "y_min"], - self.coordinates_df.loc[fov_id, "y_max"] - ), + Coordinates.X: self.coordinates_dict[fov_c_id]["xc"], + Coordinates.Y: self.coordinates_dict[fov_c_id]["yc"], } return IssCroppedBreastTile(file_path, coordinates) @@ -91,8 +87,9 @@ class ISSCroppedBreastAuxTileFetcher(TileFetcher): def __init__(self, input_dir, aux_type): self.input_dir = input_dir self.aux_type = aux_type - coordinates = os.path.join(input_dir, "coordinates.csv") - self.coordinates_df = pd.read_csv(coordinates, index_col=0) + coordinates = os.path.join(input_dir, "fabricated_test_coordinates.json") + with open(coordinates) as f: + self.coordinates_dict = json.load(f) def get_tile( self, fov_id: int, round_label: int, ch_label: int, zplane_label: int) -> FetchedTile: @@ -108,21 +105,16 @@ def get_tile( file_path = os.path.join(self.input_dir, filename) # get coordinates + fov_c_id = f"fov_{fov_id:03d}" coordinates = { - Coordinates.X: ( - self.coordinates_df.loc[fov_id, "x_min"], - self.coordinates_df.loc[fov_id, "x_max"] - ), - Coordinates.Y: ( - self.coordinates_df.loc[fov_id, "y_min"], - self.coordinates_df.loc[fov_id, "y_max"] - ), + Coordinates.X: self.coordinates_dict[fov_c_id]["xc"], + Coordinates.Y: self.coordinates_dict[fov_c_id]["yc"], } return IssCroppedBreastTile(file_path, coordinates=coordinates) -def format_data(input_dir, output_dir): +def format_data(input_dir, output_dir, num_fov): primary_image_dimensions = { Axes.ROUND: 4, @@ -145,7 +137,7 @@ def format_data(input_dir, output_dir): write_experiment_json( path=output_dir, - fov_count=16, + fov_count=num_fov, tile_format=ImageFormat.TIFF, primary_image_dimensions=primary_image_dimensions, aux_name_to_dimensions=aux_name_to_dimensions, @@ -174,6 +166,7 @@ def format_data(input_dir, output_dir): parser = argparse.ArgumentParser() parser.add_argument("input_dir", type=FsExistsType(), help=input_help_msg) parser.add_argument("output_dir", type=FsExistsType(), help=output_help_msg) + parser.add_argument("num_fov", type=int, help=fov_help_msg) args = parser.parse_args() - format_data(args.input_dir, args.output_dir) + format_data(args.input_dir, args.output_dir, args.num_fov) diff --git a/docs/source/_static/data_processing_examples/iss_pipeline.py b/docs/source/_static/data_processing_examples/iss_pipeline.py index 578785296..647647dab 100644 --- a/docs/source/_static/data_processing_examples/iss_pipeline.py +++ b/docs/source/_static/data_processing_examples/iss_pipeline.py @@ -19,7 +19,9 @@ def iss_pipeline(fov, codebook): # register the raw image learn_translation = LearnTransform.Translation(reference_stack=fov.get_image('dots'), axes=Axes.ROUND, upsampling=100) - transforms_list = learn_translation.run(primary_image.max_proj(Axes.CH, Axes.ZPLANE)) + max_projector = Filter.Reduce( + (Axes.CH, Axes.ZPLANE), func="max", module=Filter.Reduce.FunctionSource.np) + transforms_list = learn_translation.run(max_projector.run(primary_image)) warp = ApplyTransform.Warp() registered = warp.run(primary_image, transforms_list=transforms_list, in_place=False, verbose=True) diff --git a/docs/source/_static/tutorials/exec_image_manipulations.py b/docs/source/_static/tutorials/exec_image_manipulations.py index 1fbb81382..5def0282c 100644 --- a/docs/source/_static/tutorials/exec_image_manipulations.py +++ b/docs/source/_static/tutorials/exec_image_manipulations.py @@ -68,18 +68,21 @@ # 2-dimensional algorithms are typically much faster than their 3-d counterparts. # # because the Image that we've downloaded has only one :py:class:`Axes.ZPLANE`, we will instead -# demonstrate the use of :py:meth:`ImageStack.max_proj` by projecting over :py:class:`Axes.ROUND` -# to produce an image of all the spots that appear in any channel in each round. +# demonstrate the use of :py:meth:~`starfish.image.Filter.Reduce` by projecting over +# :py:class:`Axes.CH` to produce an image of all the spots that appear in any channel in each round. # -import matplotlib.pyplot as plt -import xarray as xr +from starfish.image import Filter -projected_image: starfish.ImageStack = image.max_proj(Axes.CH) +max_projector = Filter.Reduce((Axes.CH,), func="max", module=Filter.Reduce.FunctionSource.np) +projected_image: starfish.ImageStack = max_projector.run(image) ################################################################################################### # To demonstrate the effect, the below figure displays each channel of round :code:`1` in the # left and center columns, and the maximum projection on the right. +import matplotlib.pyplot as plt +import xarray as xr + # select an image for plotting in 2d round_1_ch_0: xr.DataArray = image.sel({Axes.CH: 0, Axes.ROUND: 1}).xarray.squeeze() round_1_ch_1: xr.DataArray = image.sel({Axes.CH: 1, Axes.ROUND: 1}).xarray.squeeze() diff --git a/docs/source/api/datasets/index.rst b/docs/source/api/datasets/index.rst new file mode 100644 index 000000000..9fa833a1d --- /dev/null +++ b/docs/source/api/datasets/index.rst @@ -0,0 +1,11 @@ +.. _datasets: + +Datasets +======== + +We maintain an API to let users access a handful of curated datasets for +learning how to use starfish, and benchmarking algorithms and pipelines. + + +.. automodule:: starfish.data + :members: diff --git a/docs/source/api/image/index.rst b/docs/source/api/image/index.rst index 62ede2317..ce0b3f7ec 100644 --- a/docs/source/api/image/index.rst +++ b/docs/source/api/image/index.rst @@ -23,7 +23,7 @@ Filters can be imported using ``starfish.image.Filter``, which registers all cla from starfish.image import Filter -.. autoclass:: starfish.image.Filter +.. automodule:: starfish.image.Filter :members: diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 5d5e2183a..6c299ebf3 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -26,3 +26,6 @@ API .. toctree:: utils/index.rst + +.. toctree:: + datasets/index.rst diff --git a/docs/source/getting_started/example_workflow/index.rst b/docs/source/getting_started/example_workflow/index.rst index ea1f595e4..44e00bf1f 100644 --- a/docs/source/getting_started/example_workflow/index.rst +++ b/docs/source/getting_started/example_workflow/index.rst @@ -36,7 +36,13 @@ barcode a given trace was. to create a pipeline that can be run either on the API, or using starfish's CLI. This vignette will demonstrate the API. -The above steps can be recapitulated using starfish as follows: +The above steps can be recapitulated using starfish, after downloading +an example codeblock, as follows: -.. literalinclude:: ../../_static/data_processing_examples/iss_pipeline.py +.. code-block: bash + + aws s3 cp s3://spacetx.starfish.data.public/browse/formatted/20180926/iss_breast/codebook.json iss/formatted/ \ + --no-sign-request + python docs/source/_static/data_processing_examples/iss_pipeline.py +.. literalinclude:: ../../_static/data_processing_examples/iss_pipeline.py diff --git a/docs/source/getting_started/formatting_data/advanced.rst b/docs/source/getting_started/formatting_data/advanced.rst index f40cbba8a..f22aa4d00 100644 --- a/docs/source/getting_started/formatting_data/advanced.rst +++ b/docs/source/getting_started/formatting_data/advanced.rst @@ -67,7 +67,7 @@ data, you can run the following commands: --exclude "*" \ --include "slideA_1_*" \ --include "slideA_2_*" \ - --include "coordinates.json" \ + --include "fabricated_test_coordinates.json" \ --no-sign-request ls iss/raw @@ -171,4 +171,4 @@ formatted data) and the number of fields of view to extract from the raw directo It should now be possible to fire up starfish and take a look at this data! Proceed to the next portion of the tutorial to learn more: -:ref:`Loading, interacting with, and visualizing data `. \ No newline at end of file +:ref:`Loading, interacting with, and visualizing data `. diff --git a/docs/source/getting_started/loading_data/index.rst b/docs/source/getting_started/loading_data/index.rst index cf36d94c1..c82e6f96e 100644 --- a/docs/source/getting_started/loading_data/index.rst +++ b/docs/source/getting_started/loading_data/index.rst @@ -106,8 +106,10 @@ we'll collapse all the spots across channels in each round, mimicing a "dots" im .. code-block:: python - In[11]: image.max_proj(Axes.CH) - Out[11]: + In[11]: from starfish.image import Filter + In[12]: max_projector = Filter.Reduce((Axes.CH,), func="max", module=Filter.Reduce.FunctionSource.np) + In[13]: max_projector.run(image) + Out[13]: Visualizing Data ---------------- @@ -122,10 +124,10 @@ to enable the :code:`qt` environment in IPython: .. code-block:: python - In[12]: ipython = get_ipython() - In[13]: ipython.magic("gui qt5") - In[14]: starfish.display(image) - Out[14]: + In[14]: ipython = get_ipython() + In[15]: ipython.magic("gui qt5") + In[16]: starfish.display(image) + Out[16]: Typing the above code should display an image viewer that looks something like this:, diff --git a/notebooks/BaristaSeq.ipynb b/notebooks/BaristaSeq.ipynb index 481754056..20c24000a 100644 --- a/notebooks/BaristaSeq.ipynb +++ b/notebooks/BaristaSeq.ipynb @@ -111,9 +111,9 @@ "source": [ "Project into 2D\n", "---------------\n", - "BaristaSeq is typically processed in 2d. Starfish exposes\n", - "`ImageStack.max_proj` to enable a user to max-project any axes. Here\n", - "we max project Z for both the nissl images and the primary images." + "BaristaSeq is typically processed in 2d. Starfish allows users to reduce data using arbitrary\n", + "methods via `starfish.image.Filter.Reduce`. Here we max project Z for both the nissl images and\n", + "the primary images." ] }, { @@ -122,8 +122,10 @@ "metadata": {}, "outputs": [], "source": [ - "z_projected_image = img.max_proj(Axes.ZPLANE)\n", - "z_projected_nissl = nissl.max_proj(Axes.ZPLANE)\n", + "from starfish.image import Filter\n", + "max_projector = Filter.Reduce((Axes.ZPLANE,), func=\"max\", module=Filter.Reduce.FunctionSource.np)\n", + "z_projected_image = max_projector.run(img)\n", + "z_projected_nissl = max_projector.run(nissl)\n", "\n", "# show the projected data\n", "f, (ax1, ax2) = plt.subplots(ncols=2)\n", @@ -479,4 +481,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/notebooks/ISS.ipynb b/notebooks/ISS.ipynb index 1e17bb246..d6bf6dbc8 100644 --- a/notebooks/ISS.ipynb +++ b/notebooks/ISS.ipynb @@ -132,8 +132,15 @@ "metadata": {}, "outputs": [], "source": [ + "from starfish.image import Filter\n", + "\n", + "rcz_max_projector = Filter.Reduce(\n", + " (Axes.ROUND, Axes.CH, Axes.ZPLANE,), func=\"max\", module=Filter.Reduce.FunctionSource.np)\n", + "per_round_max_projector = Filter.Reduce(\n", + " (Axes.CH, Axes.ZPLANE,), func=\"max\", module=Filter.Reduce.FunctionSource.np)\n", + "\n", "dots = fov.get_image(\"dots\")\n", - "dots_single_plane = dots.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", + "dots_single_plane = rcz_max_projector.run(dots)\n", "imshow_plane(dots_single_plane, title=\"Anchor channel, all RNA molecules\")" ] }, @@ -151,7 +158,7 @@ "outputs": [], "source": [ "nuclei = fov.get_image(\"nuclei\")\n", - "nuclei_single_plane = nuclei.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", + "nuclei_single_plane = rcz_max_projector.run(nuclei)\n", "imshow_plane(nuclei_single_plane, title=\"Nuclei (DAPI) channel\")" ] }, @@ -225,7 +232,7 @@ "from starfish.image import ApplyTransform, LearnTransform\n", "\n", "learn_translation = LearnTransform.Translation(reference_stack=dots, axes=Axes.ROUND, upsampling=1000)\n", - "transforms_list = learn_translation.run(imgs.max_proj(Axes.CH, Axes.ZPLANE))\n", + "transforms_list = learn_translation.run(per_round_max_projector.run(imgs))\n", "warp = ApplyTransform.Warp()\n", "registered_imgs = warp.run(filtered_imgs, transforms_list=transforms_list, in_place=False, verbose=True)" ] @@ -327,10 +334,10 @@ "stain_thresh = .22 # binary mask for overall cells // binarization of stain\n", "min_dist = 57\n", "\n", - "registered_mp = registered_imgs.max_proj(Axes.CH, Axes.ZPLANE).xarray.squeeze()\n", + "registered_mp = per_round_max_projector.run(registered_imgs).xarray.squeeze()\n", "stain = np.mean(registered_mp, axis=0)\n", "stain = stain/stain.max()\n", - "nuclei = nuclei.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", + "nuclei = rcz_max_projector.run(nuclei)\n", "\n", "\n", "seg = Segment.Watershed(\n", @@ -390,12 +397,10 @@ "GENE2 = 'VIM'\n", "\n", "rgb = np.zeros(registered_imgs.tile_shape + (3,))\n", - "nuclei_mp = nuclei.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", - "nuclei_numpy = nuclei_mp._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", + "nuclei_numpy = rcz_max_projector.run(nuclei)._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", "rgb[:,:,0] = nuclei_numpy\n", - "dots_mp = dots.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", - "dots_mp_numpy = dots_mp._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", - "rgb[:,:,1] = dots_mp_numpy\n", + "dots_numpy = rcz_max_projector.run(dots)._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", + "rgb[:,:,1] = dots_numpy\n", "do = rgb2gray(rgb)\n", "do = do/(do.max())\n", "\n", diff --git a/notebooks/MERFISH.ipynb b/notebooks/MERFISH.ipynb index 01fed0794..995b04d68 100644 --- a/notebooks/MERFISH.ipynb +++ b/notebooks/MERFISH.ipynb @@ -324,6 +324,9 @@ "from scipy.stats import scoreatpercentile\n", "import warnings\n", "\n", + "all_max_projector = Filter.Reduce(\n", + " (Axes.ROUND, Axes.CH, Axes.ZPLANE,), func=\"max\", module=Filter.Reduce.FunctionSource.np)\n", + "\n", "f, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))\n", "\n", "with warnings.catch_warnings():\n", @@ -334,8 +337,8 @@ " show_image(np.squeeze(prop_results.decoded_image)*(mask > 2), cmap='nipy_spectral', ax=ax1)\n", " ax1.axes.set_axis_off()\n", "\n", - " mp = filtered_imgs.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", - " mp_numpy = mp._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", + " mp_numpy = all_max_projector.run(filtered_imgs)._squeezed_numpy(\n", + " Axes.ROUND, Axes.CH, Axes.ZPLANE)\n", " clim = scoreatpercentile(mp_numpy, [0.5, 99.5])\n", " show_image(mp_numpy, clim=clim, ax=ax2)\n", "\n", diff --git a/notebooks/STARmap.ipynb b/notebooks/STARmap.ipynb index cb1663ca6..10c9c48c1 100644 --- a/notebooks/STARmap.ipynb +++ b/notebooks/STARmap.ipynb @@ -27,13 +27,14 @@ "metadata": {}, "outputs": [], "source": [ - "from pprint import pprint \n", + "from pprint import pprint\n", "\n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "\n", "import starfish\n", + "from starfish import IntensityTable\n", "import starfish.data\n", "from starfish.types import Axes\n", "from starfish.util.plot import (\n", @@ -77,10 +78,11 @@ "metadata": {}, "outputs": [], "source": [ - "ch_r_projection = stack.max_proj(Axes.CH, Axes.ROUND)\n", + "ch_r_max_projector = starfish.image.Filter.Reduce((Axes.CH, Axes.ROUND), func=\"max\")\n", + "ch_r_max_projection = ch_r_max_projector.run(stack)\n", "\n", "f = plt.figure(dpi=150)\n", - "imshow_plane(ch_r_projection, sel={Axes.ZPLANE: 15})" + "imshow_plane(ch_r_max_projection, sel={Axes.ZPLANE: 15})" ] }, { @@ -132,7 +134,8 @@ "metadata": {}, "outputs": [], "source": [ - "projection = stack.max_proj(Axes.CH, Axes.ZPLANE)\n", + "ch_z_max_projector = starfish.image.Filter.Reduce((Axes.CH, Axes.ZPLANE), func=\"max\")\n", + "projection = ch_r_max_projector.run(stack)\n", "reference_image = projection.sel({Axes.ROUND: 1})\n", "\n", "ltt = starfish.image.LearnTransform.Translation(\n", @@ -147,7 +150,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "How big are the identified translations? " + "How big are the identified translations?" ] }, { @@ -192,7 +195,7 @@ "metadata": {}, "outputs": [], "source": [ - "post_projection = stack.max_proj(Axes.CH, Axes.ZPLANE)" + "post_projection = ch_z_max_projector.run(stack)" ] }, { @@ -328,7 +331,7 @@ "metadata": {}, "outputs": [], "source": [ - "decoded = experiment.codebook.decode_per_round_max(intensities.fillna(0))\n", + "decoded = experiment.codebook.decode_per_round_max(IntensityTable(intensities.fillna(0)))\n", "decode_mask = decoded['target'] != 'nan'\n", "\n", "# %gui qt\n", diff --git a/notebooks/osmFISH.ipynb b/notebooks/osmFISH.ipynb index 6d3f82f66..900428e02 100644 --- a/notebooks/osmFISH.ipynb +++ b/notebooks/osmFISH.ipynb @@ -108,7 +108,8 @@ "metadata": {}, "outputs": [], "source": [ - "mp = imgs_ghp_laplace.max_proj(Axes.ZPLANE)" + "z_max_projector = Filter.Reduce((Axes.ZPLANE,))\n", + "mp = z_max_projector.run(imgs_ghp_laplace)" ] }, { diff --git a/notebooks/py/BaristaSeq.py b/notebooks/py/BaristaSeq.py index 2961ebbae..8842961f8 100644 --- a/notebooks/py/BaristaSeq.py +++ b/notebooks/py/BaristaSeq.py @@ -79,14 +79,16 @@ # EPY: START markdown #Project into 2D #--------------- -#BaristaSeq is typically processed in 2d. Starfish exposes -#`ImageStack.max_proj` to enable a user to max-project any axes. Here -#we max project Z for both the nissl images and the primary images. +#BaristaSeq is typically processed in 2d. Starfish allows users to reduce data using arbitrary +#methods via `starfish.image.Filter.Reduce`. Here we max project Z for both the nissl images and +#the primary images. # EPY: END markdown # EPY: START code -z_projected_image = img.max_proj(Axes.ZPLANE) -z_projected_nissl = nissl.max_proj(Axes.ZPLANE) +from starfish.image import Filter +max_projector = Filter.Reduce((Axes.ZPLANE,), func="max", module=Filter.Reduce.FunctionSource.np) +z_projected_image = max_projector.run(img) +z_projected_nissl = max_projector.run(nissl) # show the projected data f, (ax1, ax2) = plt.subplots(ncols=2) diff --git a/notebooks/py/ISS.py b/notebooks/py/ISS.py index 7da0d85fb..c6a238b1d 100644 --- a/notebooks/py/ISS.py +++ b/notebooks/py/ISS.py @@ -85,8 +85,15 @@ # EPY: END markdown # EPY: START code +from starfish.image import Filter + +rcz_max_projector = Filter.Reduce( + (Axes.ROUND, Axes.CH, Axes.ZPLANE,), func="max", module=Filter.Reduce.FunctionSource.np) +per_round_max_projector = Filter.Reduce( + (Axes.CH, Axes.ZPLANE,), func="max", module=Filter.Reduce.FunctionSource.np) + dots = fov.get_image("dots") -dots_single_plane = dots.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE) +dots_single_plane = rcz_max_projector.run(dots) imshow_plane(dots_single_plane, title="Anchor channel, all RNA molecules") # EPY: END code @@ -96,7 +103,7 @@ # EPY: START code nuclei = fov.get_image("nuclei") -nuclei_single_plane = nuclei.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE) +nuclei_single_plane = rcz_max_projector.run(nuclei) imshow_plane(nuclei_single_plane, title="Nuclei (DAPI) channel") # EPY: END code @@ -146,7 +153,7 @@ from starfish.image import ApplyTransform, LearnTransform learn_translation = LearnTransform.Translation(reference_stack=dots, axes=Axes.ROUND, upsampling=1000) -transforms_list = learn_translation.run(imgs.max_proj(Axes.CH, Axes.ZPLANE)) +transforms_list = learn_translation.run(per_round_max_projector.run(imgs)) warp = ApplyTransform.Warp() registered_imgs = warp.run(filtered_imgs, transforms_list=transforms_list, in_place=False, verbose=True) # EPY: END code @@ -211,10 +218,10 @@ stain_thresh = .22 # binary mask for overall cells // binarization of stain min_dist = 57 -registered_mp = registered_imgs.max_proj(Axes.CH, Axes.ZPLANE).xarray.squeeze() +registered_mp = per_round_max_projector.run(registered_imgs).xarray.squeeze() stain = np.mean(registered_mp, axis=0) stain = stain/stain.max() -nuclei = nuclei.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE) +nuclei = rcz_max_projector.run(nuclei) seg = Segment.Watershed( @@ -255,12 +262,10 @@ GENE2 = 'VIM' rgb = np.zeros(registered_imgs.tile_shape + (3,)) -nuclei_mp = nuclei.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE) -nuclei_numpy = nuclei_mp._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE) +nuclei_numpy = rcz_max_projector.run(nuclei)._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE) rgb[:,:,0] = nuclei_numpy -dots_mp = dots.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE) -dots_mp_numpy = dots_mp._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE) -rgb[:,:,1] = dots_mp_numpy +dots_numpy = rcz_max_projector.run(dots)._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE) +rgb[:,:,1] = dots_numpy do = rgb2gray(rgb) do = do/(do.max()) diff --git a/notebooks/py/MERFISH.py b/notebooks/py/MERFISH.py index c71f3e7b5..94b65d815 100644 --- a/notebooks/py/MERFISH.py +++ b/notebooks/py/MERFISH.py @@ -223,6 +223,9 @@ from scipy.stats import scoreatpercentile import warnings +all_max_projector = Filter.Reduce( + (Axes.ROUND, Axes.CH, Axes.ZPLANE,), func="max", module=Filter.Reduce.FunctionSource.np) + f, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5)) with warnings.catch_warnings(): @@ -233,8 +236,8 @@ show_image(np.squeeze(prop_results.decoded_image)*(mask > 2), cmap='nipy_spectral', ax=ax1) ax1.axes.set_axis_off() - mp = filtered_imgs.max_proj(Axes.ROUND, Axes.CH, Axes.ZPLANE) - mp_numpy = mp._squeezed_numpy(Axes.ROUND, Axes.CH, Axes.ZPLANE) + mp_numpy = all_max_projector.run(filtered_imgs)._squeezed_numpy( + Axes.ROUND, Axes.CH, Axes.ZPLANE) clim = scoreatpercentile(mp_numpy, [0.5, 99.5]) show_image(mp_numpy, clim=clim, ax=ax2) diff --git a/notebooks/py/STARmap.py b/notebooks/py/STARmap.py index 5169d7821..885ec4561 100644 --- a/notebooks/py/STARmap.py +++ b/notebooks/py/STARmap.py @@ -17,13 +17,14 @@ # EPY: END markdown # EPY: START code -from pprint import pprint +from pprint import pprint import matplotlib import matplotlib.pyplot as plt import numpy as np import starfish +from starfish import IntensityTable import starfish.data from starfish.types import Axes from starfish.util.plot import ( @@ -54,10 +55,11 @@ # EPY: END code # EPY: START code -ch_r_projection = stack.max_proj(Axes.CH, Axes.ROUND) +ch_r_max_projector = starfish.image.Filter.Reduce((Axes.CH, Axes.ROUND), func="max") +ch_r_max_projection = ch_r_max_projector.run(stack) f = plt.figure(dpi=150) -imshow_plane(ch_r_projection, sel={Axes.ZPLANE: 15}) +imshow_plane(ch_r_max_projection, sel={Axes.ZPLANE: 15}) # EPY: END code # EPY: START markdown @@ -93,7 +95,8 @@ # EPY: END markdown # EPY: START code -projection = stack.max_proj(Axes.CH, Axes.ZPLANE) +ch_z_max_projector = starfish.image.Filter.Reduce((Axes.CH, Axes.ZPLANE), func="max") +projection = ch_r_max_projector.run(stack) reference_image = projection.sel({Axes.ROUND: 1}) ltt = starfish.image.LearnTransform.Translation( @@ -105,7 +108,7 @@ # EPY: END code # EPY: START markdown -#How big are the identified translations? +#How big are the identified translations? # EPY: END markdown # EPY: START code @@ -129,7 +132,7 @@ # EPY: END markdown # EPY: START code -post_projection = stack.max_proj(Axes.CH, Axes.ZPLANE) +post_projection = ch_z_max_projector.run(stack) # EPY: END code # EPY: START code @@ -231,7 +234,7 @@ def plot_scaling_result( # EPY: END markdown # EPY: START code -decoded = experiment.codebook.decode_per_round_max(intensities.fillna(0)) +decoded = experiment.codebook.decode_per_round_max(IntensityTable(intensities.fillna(0))) decode_mask = decoded['target'] != 'nan' # %gui qt diff --git a/notebooks/py/osmFISH.py b/notebooks/py/osmFISH.py index d62fe72e1..79a564bb1 100644 --- a/notebooks/py/osmFISH.py +++ b/notebooks/py/osmFISH.py @@ -68,7 +68,8 @@ # EPY: END markdown # EPY: START code -mp = imgs_ghp_laplace.max_proj(Axes.ZPLANE) +z_max_projector = Filter.Reduce((Axes.ZPLANE,)) +mp = z_max_projector.run(imgs_ghp_laplace) # EPY: END code # EPY: START markdown diff --git a/notebooks/smFISH.ipynb b/notebooks/smFISH.ipynb index b9ef90f76..79f9c39e5 100644 --- a/notebooks/smFISH.ipynb +++ b/notebooks/smFISH.ipynb @@ -26,8 +26,7 @@ "project.\n", "\n", "The data consist of 45 images from 1 round, 1 channels, and 33 z-planes. Each image is\n", - "(2048, 2048) (y, x). There are no test data.\n", - "" + "(2048, 2048) (y, x). There are no test data." ] }, { diff --git a/notebooks/subdir.mk b/notebooks/subdir.mk index 85a575e11..d7efb232f 100644 --- a/notebooks/subdir.mk +++ b/notebooks/subdir.mk @@ -29,10 +29,10 @@ $(ipynb_validate_targets): validate__notebooks/%.ipynb : nbencdec encode notebooks/$*.ipynb $(TEMPFILE) diff -q <(cat notebooks/py/$*.py | egrep -v '^# EPY: stripped_notebook: ') <(cat $(TEMPFILE) | egrep -v '# EPY: stripped_notebook: ') -$(ipynb_regenerate_targets): regenerate__notebooks/%.ipynb : notebooks/py/%.py +$(ipynb_regenerate_targets): regenerate__notebooks/%.ipynb : nbencdec decode notebooks/py/$*.py notebooks/$*.ipynb -$(py_regenerate_targets): regenerate__notebooks/py/%.py : notebooks/%.ipynb +$(py_regenerate_targets): regenerate__notebooks/py/%.py : nbencdec encode notebooks/$*.ipynb notebooks/py/$*.py -.PHONY : $(py_files) +.PHONY : $(py_files) $(ipynb_regenerate_targets) $(py_regenerate_targets) diff --git a/starfish/REQUIREMENTS-STRICT.txt b/starfish/REQUIREMENTS-STRICT.txt index 91e7834fe..9069db2cc 100644 --- a/starfish/REQUIREMENTS-STRICT.txt +++ b/starfish/REQUIREMENTS-STRICT.txt @@ -3,8 +3,8 @@ appnope==0.1.0 attrs==19.1.0 backcall==0.1.0 bleach==3.1.0 -boto3==1.9.194 -botocore==1.12.194 +boto3==1.9.215 +botocore==1.12.215 certifi==2019.6.16 chardet==3.0.4 Click==7.0 @@ -13,19 +13,19 @@ dataclasses==0.6 decorator==4.4.0 defusedxml==0.6.0 diskcache==4.0.0 -docutils==0.14 +docutils==0.15.2 entrypoints==0.3 idna==2.8 imageio==2.5.0 -ipykernel==5.1.1 -ipython==7.6.1 +ipykernel==5.1.2 +ipython==7.7.0 ipython-genutils==0.2.0 -ipywidgets==7.5.0 -jedi==0.14.1 +ipywidgets==7.5.1 +jedi==0.15.1 Jinja2==2.10.1 jmespath==0.9.4 joblib==0.13.2 -jsonschema==3.0.1 +jsonschema==3.0.2 jupyter-client==5.3.1 jupyter-core==4.5.0 kiwisolver==1.1.0 @@ -33,13 +33,13 @@ MarkupSafe==1.1.1 matplotlib==3.1.1 mistune==0.8.4 mpmath==1.1.0 -nbconvert==5.5.0 +nbconvert==5.6.0 nbformat==4.4.0 networkx==2.3 -notebook==6.0.0 -numpy==1.16.4 -packaging==19.0 -pandas==0.25.0 +notebook==6.0.1 +numpy==1.17.0 +packaging==19.1 +pandas==0.25.1 pandocfilters==1.4.2 parso==0.5.1 pexpect==4.7.0 @@ -49,34 +49,34 @@ prometheus-client==0.7.1 prompt-toolkit==2.0.9 ptyprocess==0.6.0 Pygments==2.4.2 -pyparsing==2.4.0 -pyrsistent==0.15.3 +pyparsing==2.4.2 +pyrsistent==0.15.4 python-dateutil==2.8.0 -pytz==2019.1 +pytz==2019.2 PyWavelets==1.0.3 -PyYAML==5.1.1 -pyzmq==18.0.2 +PyYAML==5.1.2 +pyzmq==18.1.0 regional==1.1.2 requests==2.22.0 s3transfer==0.2.1 scikit-image==0.15.0 -scikit-learn==0.21.2 -scipy==1.3.0 +scikit-learn==0.21.3 +scipy==1.3.1 semantic-version==2.6.0 Send2Trash==1.5.0 showit==1.1.4 six==1.12.0 -slicedimage==4.0.1 +slicedimage==4.0.2 sympy==1.4 terminado==0.8.2 testpath==0.4.2 tornado==6.0.3 -tqdm==4.32.2 +tqdm==4.35.0 trackpy==0.4.1 traitlets==4.3.2 urllib3==1.25.3 -validators==0.13.0 +validators==0.14.0 wcwidth==0.1.7 webencodings==0.5.1 -widgetsnbextension==3.5.0 +widgetsnbextension==3.5.1 xarray==0.12.3 diff --git a/starfish/core/image/Filter/__init__.py b/starfish/core/image/Filter/__init__.py index b3eab6209..8e5365a66 100644 --- a/starfish/core/image/Filter/__init__.py +++ b/starfish/core/image/Filter/__init__.py @@ -1,3 +1,4 @@ +from ._base import FilterAlgorithmBase from .bandpass import Bandpass from .clip import Clip from .clip_percentile_to_zero import ClipPercentileToZero @@ -14,3 +15,11 @@ from .richardson_lucy_deconvolution import DeconvolvePSF from .white_tophat import WhiteTophat from .zero_by_channel_magnitude import ZeroByChannelMagnitude + +# autodoc's automodule directive only captures the modules explicitly listed in __all__. +all_filters = { + filter_name: filter_cls + for filter_name, filter_cls in locals().items() + if isinstance(filter_cls, type) and FilterAlgorithmBase in filter_cls.__mro__ +} +__all__ = list(all_filters.keys()) diff --git a/starfish/core/image/_registration/LearnTransform/translation.py b/starfish/core/image/_registration/LearnTransform/translation.py index be83cc0b1..477acfc0b 100644 --- a/starfish/core/image/_registration/LearnTransform/translation.py +++ b/starfish/core/image/_registration/LearnTransform/translation.py @@ -23,7 +23,9 @@ class Translation(LearnTransformBase): upsampling : int upsampling factor (default=1). See http://scikit-image.org/docs/dev/api/skimage.feature.html#skimage.feature.register_translation - for an explanation of this parameter. + for an explanation of this parameter. in brief, this parameter determines the resolution of + the registration. A value of 1 represents pixel resolution, a value of 10 is 1/10th of + a pixel, a value of 300 is 1/300th of a pixel, and so on. """ def __init__(self, reference_stack: ImageStack, axes: Axes, upsampling: int=1): diff --git a/starfish/core/spots/DetectSpots/local_search_blob_detector.py b/starfish/core/spots/DetectSpots/local_search_blob_detector.py index 45f1f5d62..b47df7d92 100644 --- a/starfish/core/spots/DetectSpots/local_search_blob_detector.py +++ b/starfish/core/spots/DetectSpots/local_search_blob_detector.py @@ -39,6 +39,30 @@ class LocalSearchBlobDetector(DetectSpotsAlgorithmBase): consensus filtering can be used to extract codes that are consistently extracted across rounds. + In brief, this spot finder operates on a few assumptions: + 1. Codes that represent transcripts are one-hot, meaning that in each round, one and only one + channel should be "on" for a given transcript. + 2. Due to experimental conditions, there may be a small amount of jitter the the exact position + of each spots. + 3. To build codes in these circumstances, one must be able to account for small deviations in + physical position to reconstruct codes. + + The LocalSearchBlobDetector accomplishes this as follows: + + 1. Identify spots independently in all rounds and channels. + 2. Identify an `anchor round`. Spots in this round will serve as the central location of a + local search. The best round to select for the anchor round is the one that has the best + signal to noise ratio. + 3. In rounds other than the anchor round, search for the closest spot that is within + `search_radius` pixels of the anchor spot. Intuitively, search radius should be the smallest + value possible that accounts for local jitter in spot position introduced by your + experimental approach. A common value is 2.5 pixels. + 4. Construct an IntensityTable, where for each anchor spot, spots in other rounds and channels + are merged into single features, storing the (z, y, x) coordinates of the anchor spots. + + The IntensityTable can then be decoded with downstream tools. This approach was used to good + effect in the STARmap paper (DOI: 10.1126/science.aat5691) + Parameters ---------- min_sigma : float diff --git a/starfish/data.py b/starfish/data.py index fc27f8f56..5b1099f8e 100644 --- a/starfish/data.py +++ b/starfish/data.py @@ -53,7 +53,7 @@ def allen_smFISH(use_test_data: bool = False) -> Experiment: def DARTFISH(use_test_data: bool = False) -> Experiment: """ Loads an experiment with a single field of view from unpublished data generated with DARTFISH v1 - 2017, produced by imaging human occipital cortex. These data were donated by the Zhuang lab as + 2017, produced by imaging human occipital cortex. These data were donated by the Zhang lab as part of the SpaceTx consortium project. The data consist of 18 images from 3 channels, 6 rounds, and 1 z-plane. Each image is (998, 998) diff --git a/workflows/wdl/iss_published/recipe.py b/workflows/wdl/iss_published/recipe.py index 94e44e614..d92f4d8a8 100644 --- a/workflows/wdl/iss_published/recipe.py +++ b/workflows/wdl/iss_published/recipe.py @@ -36,7 +36,8 @@ def process_fov(field_num: int, experiment_str: str): print("Learning Transform") learn_translation = LearnTransform.Translation(reference_stack=dots, axes=Axes.ROUND, upsampling=1000) - transforms_list = learn_translation.run(imgs.max_proj(Axes.CH, Axes.ZPLANE)) + max_projector = Filter.Reduce((Axes.CH, Axes.ZPLANE)) + transforms_list = learn_translation.run(max_projector.run(imgs)) print("Applying transform") warp = ApplyTransform.Warp() diff --git a/workflows/wdl/iss_spaceTX/recipe.py b/workflows/wdl/iss_spaceTX/recipe.py index f2f2630dd..d835a675f 100644 --- a/workflows/wdl/iss_spaceTX/recipe.py +++ b/workflows/wdl/iss_spaceTX/recipe.py @@ -24,7 +24,8 @@ def process_fov(field_num: int, experiment_str: str): fov = experiment[fov_str] imgs = fov.get_image(starfish.FieldOfView.PRIMARY_IMAGES) - dots = imgs.max_proj(Axes.CH) + max_projector = starfish.image.Filter.Reduce((Axes.CH, Axes.ZPLANE)) + dots = max_projector.run(imgs) # filter filt = starfish.image.Filter.WhiteTophat(masking_radius=15, is_volume=False)