Merge remote-tracking branch 'origin/master' into saxelrod-decoded-in…

…tensity-table

CHANGELOG.md

@@ -1,3 +1,19 @@
+## [0.1.5] - 2019-08-12
+- Update the documentation for data formatters (#1476)
+- add ability to convert segmentation masks to a label image
+- If in_place=True, we should return None (#1473)
+- downgrade pyparsing (#1467)
+- fixes unicode in issue template (#1464)
+- Adds issue templates (#1460)
+- Updating requirements. (#1461)
+- Bump to slicedimage 4.0.1 (#1458)
+- on-demand loading of data. (#1456)
+- Get rid of the check-requirements cron job. (#1448)
+- Fixing travis build  (#1457)
+- removing duplicate file (#1455)
+- Remove Cli (#1444)
+
+
 ## [0.1.4] - 2019-07-16
 - Update in-place experiment writing to use the new WriterContract API in slicedimage 4.0.0 (#1447)
 - data set formatter with fixed filenames (#1421)

REQUIREMENTS-STRICT.txt

@@ -3,8 +3,8 @@ appnope==0.1.0
 attrs==19.1.0
 backcall==0.1.0
 bleach==3.1.0
-boto3==1.9.194
-botocore==1.12.194
+boto3==1.9.215
+botocore==1.12.215
 certifi==2019.6.16
 chardet==3.0.4
 Click==7.0
@@ -13,33 +13,33 @@ dataclasses==0.6
 decorator==4.4.0
 defusedxml==0.6.0
 diskcache==4.0.0
-docutils==0.14
+docutils==0.15.2
 entrypoints==0.3
 idna==2.8
 imageio==2.5.0
-ipykernel==5.1.1
-ipython==7.6.1
+ipykernel==5.1.2
+ipython==7.7.0
 ipython-genutils==0.2.0
-ipywidgets==7.5.0
-jedi==0.14.1
+ipywidgets==7.5.1
+jedi==0.15.1
 Jinja2==2.10.1
 jmespath==0.9.4
 joblib==0.13.2
-jsonschema==3.0.1
+jsonschema==3.0.2
 jupyter-client==5.3.1
 jupyter-core==4.5.0
 kiwisolver==1.1.0
 MarkupSafe==1.1.1
 matplotlib==3.1.1
 mistune==0.8.4
 mpmath==1.1.0
-nbconvert==5.5.0
+nbconvert==5.6.0
 nbformat==4.4.0
 networkx==2.3
-notebook==6.0.0
-numpy==1.16.4
-packaging==19.0
-pandas==0.25.0
+notebook==6.0.1
+numpy==1.17.0
+packaging==19.1
+pandas==0.25.1
 pandocfilters==1.4.2
 parso==0.5.1
 pexpect==4.7.0
@@ -49,34 +49,34 @@ prometheus-client==0.7.1
 prompt-toolkit==2.0.9
 ptyprocess==0.6.0
 Pygments==2.4.2
-pyparsing==2.4.0
-pyrsistent==0.15.3
+pyparsing==2.4.2
+pyrsistent==0.15.4
 python-dateutil==2.8.0
-pytz==2019.1
+pytz==2019.2
 PyWavelets==1.0.3
-PyYAML==5.1.1
-pyzmq==18.0.2
+PyYAML==5.1.2
+pyzmq==18.1.0
 regional==1.1.2
 requests==2.22.0
 s3transfer==0.2.1
 scikit-image==0.15.0
-scikit-learn==0.21.2
-scipy==1.3.0
+scikit-learn==0.21.3
+scipy==1.3.1
 semantic-version==2.6.0
 Send2Trash==1.5.0
 showit==1.1.4
 six==1.12.0
-slicedimage==4.0.1
+slicedimage==4.0.2
 sympy==1.4
 terminado==0.8.2
 testpath==0.4.2
 tornado==6.0.3
-tqdm==4.32.2
+tqdm==4.35.0
 trackpy==0.4.1
 traitlets==4.3.2
 urllib3==1.25.3
-validators==0.13.0
+validators==0.14.0
 wcwidth==0.1.7
 webencodings==0.5.1
-widgetsnbextension==3.5.0
+widgetsnbextension==3.5.1
 xarray==0.12.3

REQUIREMENTS.txt

@@ -10,7 +10,7 @@ scikit-image>=0.14.0
 scikit-learn
 scipy
 showit >= 1.1.4
-slicedimage==4.0.1
+slicedimage==4.0.2
 scikit-learn
 sympy
 tqdm

docker/Dockerfile

@@ -43,6 +43,7 @@ USER starfish
 # Set up the initial conda environment
 COPY --chown=starfish:starfish docker/environment.yml /src/docker/environment.yml
 COPY --chown=starfish:starfish docker/pip-config /src/
+COPY --chown=starfish:starfish docker/condarc /home/starfish/.condarc
 COPY --chown=starfish:starfish REQUIREMENTS* /src/
 WORKDIR /src
 ENV PIP_CONFIG_FILE=/src/pip-config

docker/condarc

@@ -0,0 +1,2 @@
+pkgs_dirs:
+ - /home/starfish/.conda/pkgs

docs/source/_static/data_formatting_examples/format_iss_breast_data.py

@@ -5,11 +5,11 @@
 The following script formats In-Situ Sequencing data in SpaceTx-Format
 """
 import argparse
+import json
 import os
 from typing import Mapping, Union
 
 import numpy as np
-import pandas as pd
 from skimage.io import imread
 from slicedimage import ImageFormat
 
@@ -48,8 +48,9 @@ def tile_data(self) -> np.ndarray:
 class ISSCroppedBreastPrimaryTileFetcher(TileFetcher):
     def __init__(self, input_dir):
         self.input_dir = input_dir
-        coordinates = os.path.join(input_dir, "coordinates.csv")
-        self.coordinates_df = pd.read_csv(coordinates, index_col=0)
+        coordinates = os.path.join(input_dir, "fabricated_test_coordinates.json")
+        with open(coordinates) as f:
+            self.coordinates_dict = json.load(f)
 
     @property
     def ch_dict(self):
@@ -73,15 +74,10 @@ def get_tile(
         file_path = os.path.join(self.input_dir, filename)
 
         # get coordinates
+        fov_c_id = f"fov_{fov_id:03d}"
         coordinates = {
-            Coordinates.X: (
-                self.coordinates_df.loc[fov_id, "x_min"],
-                self.coordinates_df.loc[fov_id, "x_max"]
-            ),
-            Coordinates.Y: (
-                self.coordinates_df.loc[fov_id, "y_min"],
-                self.coordinates_df.loc[fov_id, "y_max"]
-            ),
+            Coordinates.X: self.coordinates_dict[fov_c_id]["xc"],
+            Coordinates.Y: self.coordinates_dict[fov_c_id]["yc"],
         }
 
         return IssCroppedBreastTile(file_path, coordinates)
@@ -91,8 +87,9 @@ class ISSCroppedBreastAuxTileFetcher(TileFetcher):
     def __init__(self, input_dir, aux_type):
         self.input_dir = input_dir
         self.aux_type = aux_type
-        coordinates = os.path.join(input_dir, "coordinates.csv")
-        self.coordinates_df = pd.read_csv(coordinates, index_col=0)
+        coordinates = os.path.join(input_dir, "fabricated_test_coordinates.json")
+        with open(coordinates) as f:
+            self.coordinates_dict = json.load(f)
 
     def get_tile(
             self, fov_id: int, round_label: int, ch_label: int, zplane_label: int) -> FetchedTile:
@@ -108,21 +105,16 @@ def get_tile(
         file_path = os.path.join(self.input_dir, filename)
 
         # get coordinates
+        fov_c_id = f"fov_{fov_id:03d}"
         coordinates = {
-            Coordinates.X: (
-                self.coordinates_df.loc[fov_id, "x_min"],
-                self.coordinates_df.loc[fov_id, "x_max"]
-            ),
-            Coordinates.Y: (
-                self.coordinates_df.loc[fov_id, "y_min"],
-                self.coordinates_df.loc[fov_id, "y_max"]
-            ),
+            Coordinates.X: self.coordinates_dict[fov_c_id]["xc"],
+            Coordinates.Y: self.coordinates_dict[fov_c_id]["yc"],
         }
 
         return IssCroppedBreastTile(file_path, coordinates=coordinates)
 
 
-def format_data(input_dir, output_dir):
+def format_data(input_dir, output_dir, num_fov):
 
     primary_image_dimensions = {
         Axes.ROUND: 4,
@@ -145,7 +137,7 @@ def format_data(input_dir, output_dir):
 
     write_experiment_json(
         path=output_dir,
-        fov_count=16,
+        fov_count=num_fov,
         tile_format=ImageFormat.TIFF,
         primary_image_dimensions=primary_image_dimensions,
         aux_name_to_dimensions=aux_name_to_dimensions,
@@ -174,6 +166,7 @@ def format_data(input_dir, output_dir):
     parser = argparse.ArgumentParser()
     parser.add_argument("input_dir", type=FsExistsType(), help=input_help_msg)
     parser.add_argument("output_dir", type=FsExistsType(), help=output_help_msg)
+    parser.add_argument("num_fov", type=int, help=fov_help_msg)
 
     args = parser.parse_args()
-    format_data(args.input_dir, args.output_dir)
+    format_data(args.input_dir, args.output_dir, args.num_fov)

docs/source/_static/data_processing_examples/iss_pipeline.py

@@ -19,7 +19,9 @@ def iss_pipeline(fov, codebook):
     # register the raw image
     learn_translation = LearnTransform.Translation(reference_stack=fov.get_image('dots'),
                                                    axes=Axes.ROUND, upsampling=100)
-    transforms_list = learn_translation.run(primary_image.max_proj(Axes.CH, Axes.ZPLANE))
+    max_projector = Filter.Reduce(
+        (Axes.CH, Axes.ZPLANE), func="max", module=Filter.Reduce.FunctionSource.np)
+    transforms_list = learn_translation.run(max_projector.run(primary_image))
     warp = ApplyTransform.Warp()
     registered = warp.run(primary_image, transforms_list=transforms_list,  in_place=False, verbose=True)
 

docs/source/_static/tutorials/exec_image_manipulations.py

@@ -68,18 +68,21 @@
 # 2-dimensional algorithms are typically much faster than their 3-d counterparts.
 #
 # because the Image that we've downloaded has only one :py:class:`Axes.ZPLANE`, we will instead
-# demonstrate the use of :py:meth:`ImageStack.max_proj` by projecting over :py:class:`Axes.ROUND`
-# to produce an image of all the spots that appear in any channel in each round.
+# demonstrate the use of :py:meth:~`starfish.image.Filter.Reduce` by projecting over
+# :py:class:`Axes.CH` to produce an image of all the spots that appear in any channel in each round.
 #
-import matplotlib.pyplot as plt
-import xarray as xr
+from starfish.image import Filter
 
-projected_image: starfish.ImageStack = image.max_proj(Axes.CH)
+max_projector = Filter.Reduce((Axes.CH,), func="max", module=Filter.Reduce.FunctionSource.np)
+projected_image: starfish.ImageStack = max_projector.run(image)
 
 ###################################################################################################
 # To demonstrate the effect, the below figure displays each channel of round :code:`1` in the
 # left and center columns, and the maximum projection on the right.
 
+import matplotlib.pyplot as plt
+import xarray as xr
+
 # select an image for plotting in 2d
 round_1_ch_0: xr.DataArray = image.sel({Axes.CH: 0, Axes.ROUND: 1}).xarray.squeeze()
 round_1_ch_1: xr.DataArray = image.sel({Axes.CH: 1, Axes.ROUND: 1}).xarray.squeeze()

docs/source/api/datasets/index.rst

@@ -0,0 +1,11 @@
+.. _datasets:
+
+Datasets
+========
+
+We maintain an API to let users access a handful of curated datasets for
+learning how to use starfish, and benchmarking algorithms and pipelines.
+
+
+.. automodule:: starfish.data
+   :members:

docs/source/api/image/index.rst

@@ -23,7 +23,7 @@ Filters can be imported using ``starfish.image.Filter``, which registers all cla
 
     from starfish.image import Filter
 
-.. autoclass:: starfish.image.Filter
+.. automodule:: starfish.image.Filter
    :members:
 
 

docs/source/api/index.rst

@@ -26,3 +26,6 @@ API
 
 .. toctree::
    utils/index.rst
+
+.. toctree::
+  datasets/index.rst

docs/source/getting_started/example_workflow/index.rst

@@ -36,7 +36,13 @@ barcode a given trace was.
 to create a pipeline that can be run either on the API, or using starfish's CLI. This vignette will
 demonstrate the API.
 
-The above steps can be recapitulated using starfish as follows:
+The above steps can be recapitulated using starfish, after downloading
+an example codeblock, as follows:
 
-.. literalinclude:: ../../_static/data_processing_examples/iss_pipeline.py
+.. code-block: bash
+
+    aws s3 cp s3://spacetx.starfish.data.public/browse/formatted/20180926/iss_breast/codebook.json iss/formatted/ \
+        --no-sign-request
+   python docs/source/_static/data_processing_examples/iss_pipeline.py
 
+.. literalinclude:: ../../_static/data_processing_examples/iss_pipeline.py

docs/source/getting_started/formatting_data/advanced.rst

@@ -67,7 +67,7 @@ data, you can run the following commands:
         --exclude "*" \
         --include "slideA_1_*" \
         --include "slideA_2_*" \
-        --include "coordinates.json" \
+        --include "fabricated_test_coordinates.json" \
         --no-sign-request
     ls iss/raw
 
@@ -171,4 +171,4 @@ formatted data) and the number of fields of view to extract from the raw directo
 
 It should now be possible to fire up starfish and take a look at this data! Proceed to the next
 portion of the tutorial to learn more:
-:ref:`Loading, interacting with, and visualizing data <loading_data>`.
+:ref:`Loading, interacting with, and visualizing data <loading_data>`.

docs/source/getting_started/loading_data/index.rst

@@ -106,8 +106,10 @@ we'll collapse all the spots across channels in each round, mimicing a "dots" im
 
 .. code-block:: python
 
-    In[11]: image.max_proj(Axes.CH)
-    Out[11]: <starfish.ImageStack (r: 4, c: 1, z: 1, y: 1000, x: 1000)>
+    In[11]: from starfish.image import Filter
+    In[12]: max_projector = Filter.Reduce((Axes.CH,), func="max", module=Filter.Reduce.FunctionSource.np)
+    In[13]: max_projector.run(image)
+    Out[13]: <starfish.ImageStack (r: 4, c: 1, z: 1, y: 1000, x: 1000)>
 
 Visualizing Data
 ----------------
@@ -122,10 +124,10 @@ to enable the :code:`qt` environment in IPython:
 
 .. code-block:: python
 
-    In[12]: ipython = get_ipython()
-    In[13]: ipython.magic("gui qt5")
-    In[14]: starfish.display(image)
-    Out[14]: <napari.components._viewer.model.Viewer at 0x15f7b44e0>
+    In[14]: ipython = get_ipython()
+    In[15]: ipython.magic("gui qt5")
+    In[16]: starfish.display(image)
+    Out[16]: <napari.components._viewer.model.Viewer at 0x15f7b44e0>
 
 Typing the above code should display an image viewer that looks something like this:,