Remove .tif/.tiff extension from list_files for MIBItiff reading in Segment_Image_Data.ipynb

ark/utils/deepcell_service_utils.py

@@ -6,6 +6,8 @@
 from zipfile import ZipFile
 import warnings
 
+from ark.utils import misc_utils
+
 
 def create_deepcell_output(deepcell_input_dir, deepcell_output_dir, fovs=None,
                            suffix='_feature_0', host='https://deepcell.org', job_type='multiplex'):
@@ -37,30 +39,49 @@ def create_deepcell_output(deepcell_input_dir, deepcell_output_dir, fovs=None,
             ValueError:
                 Raised if there is some fov X (from fovs list) s.t.
                 the file <deepcell_input_dir>/fovX.tif does not exist
-        """
+    """
+
+    # extract all the files from deepcell_input_dir
+    input_files = io_utils.list_files(deepcell_input_dir, substrs=['.tif', '.tiff'])
+
+    # set fovs equal to input_files it not already set
     if fovs is None:
-        tifs = io_utils.list_files(deepcell_input_dir, substrs='.tif')
-        fovs = io_utils.extract_delimited_names(tifs, delimiter='.')
+        fovs = input_files
+
+    # now extract only the names of the fovs without the file extension
+    fovs = io_utils.extract_delimited_names(fovs, delimiter='.')
+
+    # make sure that all fovs actually exist in the list of input_files
+    misc_utils.verify_in_list(
+        fovs=fovs,
+        deepcell_input_files=io_utils.extract_delimited_names(input_files, delimiter='.'))
 
+    # define the location of the zip file for our fovs
     zip_path = os.path.join(deepcell_input_dir, 'fovs.zip')
     if os.path.isfile(zip_path):
-        warnings.warn(f'{zip_path} will be overwritten.')
+        print(f'{zip_path} will be overwritten')
+        warnings.warn('overwriting')
 
+    # write all files to the zip file
     print('Zipping preprocessed tif files.')
+
     with ZipFile(zip_path, 'w') as zipObj:
         for fov in fovs:
-            filename = os.path.join(deepcell_input_dir, fov + '.tif')
-            if not os.path.isfile(filename):
-                raise ValueError('Could not find .tif file for %s. '
-                                 'Invalid value for %s' % (fov, filename))
+            # file has .tif extension
+            if fov + '.tif' in input_files:
+                filename = os.path.join(deepcell_input_dir, fov + '.tif')
+            # file has .tiff extension
+            else:
+                filename = os.path.join(deepcell_input_dir, fov + '.tiff')
+
             zipObj.write(filename, os.path.basename(filename))
 
-    print('Uploading files to DeppCell server.')
+    # pass the zip file to deepcell.org
+    print('Uploading files to DeepCell server.')
     run_deepcell_task(zip_path, deepcell_output_dir, host, job_type)
-    os.remove(zip_path)
 
-    print('Extracting tif files from DeepCell response.')
     # extract the .tif output
+    print('Extracting tif files from DeepCell response.')
     zip_files = glob.glob(os.path.join(deepcell_output_dir, '*.zip'))
     zip_files.sort(key=os.path.getmtime)
     with ZipFile(zip_files[-1], 'r') as zipObj:
@@ -87,7 +108,7 @@ def run_deepcell_task(input_dir, output_dir, host='https://deepcell.org',
             job_type (str):
                 Name of job workflow (multiplex, segmentation, tracking).
                 Default: 'multiplex'
-        """
+    """
 
     mgr_kwargs = {
         'host': host,

ark/utils/deepcell_service_utils_test.py

@@ -10,10 +10,11 @@
 def mocked_run_deepcell(input_dir, output_dir, host, job_type):
     pathlib.Path(os.path.join(output_dir, 'fov1_feature_0.tif')).touch()
     pathlib.Path(os.path.join(output_dir, 'fov2_feature_0.tif')).touch()
+    pathlib.Path(os.path.join(output_dir, 'fov3_feature_0.tif')).touch()
 
     zip_path = os.path.join(output_dir, 'example_output.zip')
     with ZipFile(zip_path, 'w') as zipObj:
-        for i in [1, 2]:
+        for i in range(1, 4):
             filename = os.path.join(output_dir, f'fov{i}_feature_0.tif')
             zipObj.write(filename, os.path.basename(filename))
             os.remove(filename)
@@ -25,12 +26,19 @@ def test_create_deepcell_output(mocker):
 
         input_dir = os.path.join(temp_dir, 'input_dir')
         os.makedirs(input_dir)
+        pathlib.Path(os.path.join(input_dir, 'fov1.tif')).touch()
+        pathlib.Path(os.path.join(input_dir, 'fov2.tif')).touch()
+        pathlib.Path(os.path.join(input_dir, 'fov3.tiff')).touch()
 
         output_dir = os.path.join(temp_dir, 'output_dir')
         os.makedirs(output_dir)
-        pathlib.Path(os.path.join(input_dir, 'fov1.tif')).touch()
-        pathlib.Path(os.path.join(input_dir, 'fov2.tif')).touch()
 
+        with pytest.raises(ValueError):
+            # fail if non-existent fovs are specified
+            create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir,
+                                   fovs=['fov1', 'fov1000'])
+
+        # test with specified fov list
         create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir,
                                fovs=['fov1', 'fov2'])
 
@@ -41,18 +49,40 @@ def test_create_deepcell_output(mocker):
         assert os.path.exists(os.path.join(output_dir, 'fov1_feature_0.tif'))
         assert os.path.exists(os.path.join(output_dir, 'fov2_feature_0.tif'))
 
-        # if fovs is None, all .tif files in input dir should be taken
         os.remove(os.path.join(output_dir, 'fov1_feature_0.tif'))
         os.remove(os.path.join(output_dir, 'fov2_feature_0.tif'))
-        create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir)
+        os.remove(os.path.join(output_dir, 'example_output.zip'))
+
+        # test with mixed fov/file list
+        create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir,
+                               fovs=['fov1', 'fov2.tif', 'fov3.tiff'])
+
+        # make sure DeepCell (.zip) output exists
+        assert os.path.exists(os.path.join(output_dir, 'example_output.zip'))
+
+        # DeepCell output .zip file should be extracted
         assert os.path.exists(os.path.join(output_dir, 'fov1_feature_0.tif'))
         assert os.path.exists(os.path.join(output_dir, 'fov2_feature_0.tif'))
+        assert os.path.exists(os.path.join(output_dir, 'fov3_feature_0.tif'))
+
+        os.remove(os.path.join(output_dir, 'fov1_feature_0.tif'))
+        os.remove(os.path.join(output_dir, 'fov2_feature_0.tif'))
+        os.remove(os.path.join(output_dir, 'fov3_feature_0.tif'))
+        os.remove(os.path.join(output_dir, 'example_output.zip'))
+
+        # if fovs is None, all .tif files in input dir should be taken
+        create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir)
 
-        # /fovs.zip file should be removed from input folder after completion
-        assert not os.path.isfile(os.path.join(input_dir, 'fovs.zip'))
+        # make sure DeepCell (.zip) output exists
+        assert os.path.exists(os.path.join(output_dir, 'example_output.zip'))
+
+        assert os.path.exists(os.path.join(output_dir, 'fov1_feature_0.tif'))
+        assert os.path.exists(os.path.join(output_dir, 'fov2_feature_0.tif'))
+        assert os.path.exists(os.path.join(output_dir, 'fov3_feature_0.tif'))
 
         pathlib.Path(os.path.join(input_dir, 'fovs.zip')).touch()
-        # Warning should be displayed if /fovs.zip file exists (will be overwritten)
+
+        # Warning should be displayed if fovs.zip file exists (will be overwritten)
         with pytest.warns(UserWarning):
             create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir,
                                    fovs=['fov1'])
@@ -61,12 +91,13 @@ def test_create_deepcell_output(mocker):
         with pytest.warns(UserWarning):
             create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir,
                                    suffix='_other_suffix', fovs=['fov1'])
-        pathlib.Path(os.path.join(input_dir, 'fov3.tif')).touch()
+
+        pathlib.Path(os.path.join(input_dir, 'fov4.tif')).touch()
         with pytest.warns(UserWarning):
             create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir,
-                                   fovs=['fov1', 'fov2', 'fov3'])
+                                   fovs=['fov1', 'fov2', 'fov3', 'fov4'])
 
         # ValueError should be raised if .tif file does not exists for some fov in fovs
         with pytest.raises(ValueError):
             create_deepcell_output(deepcell_input_dir=input_dir, deepcell_output_dir=output_dir,
-                                   fovs=['fov1', 'fov4'])
+                                   fovs=['fov1', 'fov5'])