Fix problems with missing groups/metadata and sparse plates

src/main/java/com/glencoesoftware/bioformats2raw/Converter.java

@@ -1338,7 +1338,6 @@ private void saveHCSMetadata(IMetadata meta) throws IOException {
         String rowName = String.valueOf(r);
         row.put("name", rowName);
         rows.add(row);
-        root.createSubGroup(rowName);
       }
     }
     catch (NullPointerException e) {
@@ -1363,7 +1362,9 @@ private void saveHCSMetadata(IMetadata meta) throws IOException {
       acquisition.put("id", String.valueOf(pa));
       acquisitions.add(acquisition);
     }
-    plateMap.put("acquisitions", acquisitions);
+    if (acquisitions.size() > 0) {
+      plateMap.put("acquisitions", acquisitions);
+    }
 
     List<Map<String, Object>> wells = new ArrayList<Map<String, Object>>();
     int maxField = Integer.MIN_VALUE;
@@ -1377,7 +1378,10 @@ private void saveHCSMetadata(IMetadata meta) throws IOException {
 
           List<Map<String, Object>> imageList =
             new ArrayList<Map<String, Object>>();
-          ZarrGroup wellGroup = root.createSubGroup(wellPath);
+          String rowPath = index.getRowPath();
+          ZarrGroup rowGroup = root.createSubGroup(rowPath);
+          String columnPath = index.getColumnPath();
+          ZarrGroup columnGroup = rowGroup.createSubGroup(columnPath);
           for (HCSIndex field : hcsIndexes) {
             if (field.getPlateIndex() == index.getPlateIndex() &&
               field.getWellRowIndex() == index.getWellRowIndex() &&
@@ -1393,9 +1397,9 @@ private void saveHCSMetadata(IMetadata meta) throws IOException {
 
           Map<String, Object> wellMap = new HashMap<String, Object>();
           wellMap.put("images", imageList);
-          Map<String, Object> attributes = wellGroup.getAttributes();
+          Map<String, Object> attributes = columnGroup.getAttributes();
           attributes.put("well", wellMap);
-          wellGroup.writeAttributes(attributes);
+          columnGroup.writeAttributes(attributes);
 
           // make sure the row/column indexes are added to the plate attributes
           // this is necessary when Plate.Rows or Plate.Columns is not set

src/main/java/com/glencoesoftware/bioformats2raw/HCSIndex.java

@@ -134,11 +134,25 @@ public void setFieldIndex(int fieldIndex) {
     this.field = fieldIndex;
   }
 
+  /**
+   * @return row path relative to the plate group
+   */
+  public String getRowPath() {
+    return String.format("%d", getWellRowIndex());
+  }
+
+  /**
+   * @return column path relative to the row group
+   */
+  public String getColumnPath() {
+    return String.format("%d", getWellColumnIndex());
+  }
+
   /**
    * @return well path relative to the plate group
    */
   public String getWellPath() {
-    return String.format("%d/%d", getWellRowIndex(), getWellColumnIndex());
+    return String.format("%s/%s", getRowPath(), getColumnPath());
   }
 
   @Override

src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java

@@ -42,6 +42,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -822,7 +823,11 @@ public void testHCSMetadata() throws Exception {
     int rowCount = 2;
     int colCount = 3;
     int fieldCount = 2;
-    checkPlateGroupLayout(output, rowCount, colCount, fieldCount, 512, 512);
+    Map<String, List<String>> plateMap = new HashMap<String, List<String>>();
+    plateMap.put("0", Arrays.asList("0", "1", "2"));
+    plateMap.put("1", Arrays.asList("0", "1", "2"));
+    checkPlateGroupLayout(output, rowCount, colCount,
+      plateMap, fieldCount, 512, 512);
 
     // check plate/well level metadata
     Map<String, Object> plate =
@@ -868,6 +873,62 @@ public void testHCSMetadata() throws Exception {
     assertEquals(1, ome.sizeOfPlateList());
   }
 
+  /**
+   * Convert a plate with default options.
+   * The output should be compliant with OME Zarr HCS.
+   */
+  @Test
+  public void testHCSMetadataNoAcquisitions() throws Exception {
+    input = getTestFile("A1-B1-only-no-acqs.xml");
+    assertTool();
+
+    ZarrGroup z = ZarrGroup.open(output);
+
+    int rowCount = 8;
+    int colCount = 12;
+    int fieldCount = 1;
+
+    // Only two rows are filled out with one column each (two Wells total)
+    Map<String, List<String>> plateMap = new HashMap<String, List<String>>();
+    plateMap.put("0", Arrays.asList("0"));
+    plateMap.put("1", Arrays.asList("0"));
+    checkPlateGroupLayout(output, rowCount, colCount,
+      plateMap, fieldCount, 2, 2);
+
+    // check plate/well level metadata
+    Map<String, Object> plate =
+        (Map<String, Object>) z.getAttributes().get("plate");
+    assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue());
+
+    List<Map<String, Object>> rows =
+      (List<Map<String, Object>>) plate.get("rows");
+    List<Map<String, Object>> columns =
+      (List<Map<String, Object>>) plate.get("columns");
+    List<Map<String, Object>> wells =
+      (List<Map<String, Object>>) plate.get("wells");
+
+    assertFalse(plate.containsKey("acquisitions"));
+
+    checkDimension(rows, rowCount);
+    checkDimension(columns, colCount);
+
+    assertEquals(2, wells.size());
+    for (Map<String, Object> well : wells) {
+      int row = ((Number) well.get("row_index")).intValue();
+      int col = ((Number) well.get("column_index")).intValue();
+
+      String wellPath = well.get("path").toString();
+      assertEquals(row + "/" + col, wellPath);
+
+      ZarrGroup wellGroup = ZarrGroup.open(output.resolve(wellPath));
+      checkWell(wellGroup, fieldCount, -1);
+    }
+
+    // check OME metadata
+    OME ome = getOMEMetadata();
+    assertEquals(1, ome.sizeOfPlateList());
+  }
+
   /**
    * 96 well plate with only well E6.
    */
@@ -882,6 +943,12 @@ public void testSingleWell() throws IOException {
     int colCount = 12;
     int fieldCount = 1;
 
+    Map<String, List<String>> plateMap = new HashMap<String, List<String>>();
+    plateMap.put("4", Arrays.asList("5"));
+
+    checkPlateGroupLayout(output, rowCount, colCount,
+      plateMap, fieldCount, 2, 2);
+
     // check plate/well level metadata
     Map<String, Object> plate =
         (Map<String, Object>) z.getAttributes().get("plate");
@@ -928,6 +995,12 @@ public void testTwoWells() throws IOException {
     int colCount = 12;
     int fieldCount = 1;
 
+    Map<String, List<String>> plateMap = new HashMap<String, List<String>>();
+    plateMap.put("2", Arrays.asList("3"));
+    plateMap.put("7", Arrays.asList("1"));
+    checkPlateGroupLayout(output, rowCount, colCount,
+      plateMap, fieldCount, 2, 2);
+
     // check plate/well level metadata
     Map<String, Object> plate =
         (Map<String, Object>) z.getAttributes().get("plate");
@@ -980,6 +1053,12 @@ public void testOnePlateRow() throws IOException {
     int colCount = 12;
     int fieldCount = 1;
 
+    Map<String, List<String>> plateMap = new HashMap<String, List<String>>();
+    plateMap.put("5", Arrays.asList(
+      "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"));
+    checkPlateGroupLayout(output, rowCount, colCount,
+      plateMap, fieldCount, 2, 2);
+
     // check plate/well level metadata
     Map<String, Object> plate =
         (Map<String, Object>) z.getAttributes().get("plate");
@@ -1206,29 +1285,60 @@ public void testChunkSizeToBig() throws Exception {
     }
   }
 
+  /**
+   * @param root dataset root path
+   * @param rowCount total rows the plate could contain
+   * @param colCount total columns the plate could contain
+   * @param validPaths array of row/column paths containing data
+   * @param fieldCount number of fields per well
+   * @param x image width
+   * @param y image height
+   */
   private void checkPlateGroupLayout(Path root, int rowCount, int colCount,
-    int fieldCount, int x, int y)
+    Map<String, List<String>> validPaths, int fieldCount, int x, int y)
     throws IOException
   {
     // check valid group layout
-    // OME (OME-XML metadata), .zattrs (Plate), .zgroup (Plate) and rows
-    assertEquals(rowCount + 3, Files.list(root).toArray().length);
+    // OME (OME-XML metadata) folder, .zattrs (Plate), .zgroup (Plate) and rows
+    assertEquals(validPaths.size() + 3, Files.list(root).toArray().length);
+
+    // for every row in the overall plate,
+    // if there are any wells with data in the row, the row path must exist
+    // if there are no wells with data in the row, the row path cannot exist
     for (int row=0; row<rowCount; row++) {
-      Path rowPath = root.resolve(Integer.toString(row));
-      // .zgroup (Row) and columns
-      assertEquals(colCount + 1, Files.list(rowPath).toArray().length);
-      for (int col=0; col<colCount; col++) {
-        Path colPath = rowPath.resolve(Integer.toString(col));
-        ZarrGroup colGroup = ZarrGroup.open(colPath);
-        // .zattrs (Column/Image), .zgroup (Column/Image) and fields
-        assertEquals(fieldCount + 2, Files.list(colPath).toArray().length);
-        for (int field=0; field<fieldCount; field++) {
-          // append resolution index
-          ZarrArray series0 = colGroup.openArray(field + "/0");
-          assertArrayEquals(new int[] {1, 1, 1, y, x}, series0.getShape());
-          assertArrayEquals(new int[] {1, 1, 1, y, x}, series0.getChunks());
+      String rowString = Integer.toString(row);
+      if (validPaths.containsKey(rowString)) {
+        Path rowPath = root.resolve(rowString);
+        // .zgroup (Row) and columns
+        List<String> validColumns = validPaths.get(rowString);
+        assertEquals(validColumns.size() + 1,
+          Files.list(rowPath).toArray().length);
+
+        // for every column in the overall plate,
+        // if this row/column is a well with data, the column path must exist
+        // otherwise, the column path cannot exist
+        for (int col=0; col<colCount; col++) {
+          String colString = Integer.toString(col);
+          if (validColumns.contains(colString)) {
+            Path colPath = rowPath.resolve(colString);
+            ZarrGroup colGroup = ZarrGroup.open(colPath);
+            // .zattrs (Column/Image), .zgroup (Column/Image) and fields
+            assertEquals(fieldCount + 2, Files.list(colPath).toArray().length);
+            for (int field=0; field<fieldCount; field++) {
+              // append resolution index
+              ZarrArray series0 = colGroup.openArray(field + "/0");
+              assertArrayEquals(new int[] {1, 1, 1, y, x}, series0.getShape());
+              assertArrayEquals(new int[] {1, 1, 1, y, x}, series0.getChunks());
+            }
+          }
+          else {
+            assertFalse(rowPath.resolve(colString).toFile().exists());
+          }
         }
       }
+      else {
+        assertFalse(root.resolve(rowString).toFile().exists());
+      }
     }
   }
 
@@ -1243,6 +1353,13 @@ private void checkDimension(List<Map<String, Object>> dims, int dimCount)
 
   private void checkWell(ZarrGroup wellGroup, int fieldCount)
     throws IOException
+  {
+    checkWell(wellGroup, fieldCount, 0);
+  }
+
+
+  private void checkWell(ZarrGroup wellGroup, int fieldCount, int acquisition)
+    throws IOException
   {
     Map<String, Object> well =
         (Map<String, Object>) wellGroup.getAttributes().get("well");
@@ -1253,7 +1370,7 @@ private void checkWell(ZarrGroup wellGroup, int fieldCount)
     for (int i=0; i<fieldCount; i++) {
       Map<String, Object> field = images.get(i);
       assertEquals(field.get("path"), String.valueOf(i));
-      assertEquals(0, field.get("acquisition"));
+      assertEquals(acquisition, field.get("acquisition"));
     }
   }
 

src/test/resources/com/glencoesoftware/bioformats2raw/test/A1-B1-only-no-acqs.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" Creator="OME Bio-Formats 6.6.1-SNAPSHOT" xsi:schemaLocation="http://www.openmicroscopy.org/Schemas/OME/2016-06 http://www.openmicroscopy.org/Schemas/OME/2016-06/ome.xsd">
+<Plate ColumnNamingConvention="number" Columns="12" ExternalIdentifier="External Identifier" ID="Plate:0" Name="Plate Name 0" RowNamingConvention="letter" Rows="8" Status="Plate status" WellOriginX="0.0" WellOriginXUnit="µm" WellOriginY="1.0" WellOriginYUnit="µm">
+<Description>Plate 0 of 1</Description>
+<Well Color="255" Column="0" ExternalDescription="External Description" ExternalIdentifier="External Identifier" ID="Well:0_0_0_0" Row="0" Type="Transfection: done">
+<WellSample ID="WellSample:0_0_0_0_0_0" Index="0" PositionX="0.0" PositionXUnit="reference frame" PositionY="1.0" PositionYUnit="reference frame" Timepoint="2006-05-04T18:13:51">
+<ImageRef ID="Image:0"></ImageRef></WellSample></Well>
+<Well Color="255" Column="0" ExternalDescription="External Description" ExternalIdentifier="External Identifier" ID="Well:0_0_1_0" Row="1" Type="Transfection: done">
+<WellSample ID="WellSample:0_0_1_0_0_0" Index="1" PositionX="0.0" PositionXUnit="reference frame" PositionY="1.0" PositionYUnit="reference frame" Timepoint="2006-05-04T18:13:51">
+<ImageRef ID="Image:1"></ImageRef></WellSample></Well>
+</Plate>
+<Image ID="Image:0" Name="test">
+<Description>Image Description 0</Description>
+<Pixels BigEndian="false" DimensionOrder="XYZCT" ID="Pixels:0" Interleaved="false" PhysicalSizeX="1.0" PhysicalSizeXUnit="µm" PhysicalSizeY="1.0" PhysicalSizeYUnit="µm" PhysicalSizeZ="1.0" PhysicalSizeZUnit="µm" SignificantBits="8" SizeC="1" SizeT="1" SizeX="2" SizeY="2" SizeZ="1" Type="uint8">
+<Channel AcquisitionMode="FluorescenceLifetime" Color="1687603455" ContrastMethod="Brightfield" EmissionWavelength="300.3" EmissionWavelengthUnit="nm" ExcitationWavelength="400.3" ExcitationWavelengthUnit="nm" Fluor="Fluor" ID="Channel:0:0" IlluminationType="Oblique" NDFilter="1.0" Name="Name" PinholeSize="0.5" PinholeSizeUnit="µm" PockelCellSetting="0" SamplesPerPixel="1"/>
+<Plane DeltaT="0.1" DeltaTUnit="s" ExposureTime="10.0" ExposureTimeUnit="s" PositionX="1.0" PositionXUnit="reference frame" PositionY="1.0" PositionYUnit="reference frame" PositionZ="1.0" PositionZUnit="reference frame" TheC="0" TheT="0" TheZ="0"/>
+<BinData xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06" Length="4" BigEndian="false">AAAAAA==</BinData></Pixels></Image>
+<Image ID="Image:1" Name="test 2">
+<Description>Image Description 1</Description>
+<Pixels BigEndian="false" DimensionOrder="XYZCT" ID="Pixels:1" Interleaved="false" PhysicalSizeX="1.0" PhysicalSizeXUnit="µm" PhysicalSizeY="1.0" PhysicalSizeYUnit="µm" PhysicalSizeZ="1.0" PhysicalSizeZUnit="µm" SignificantBits="8" SizeC="1" SizeT="1" SizeX="2" SizeY="2" SizeZ="1" Type="uint8">
+<Channel AcquisitionMode="FluorescenceLifetime" Color="1687603455" ContrastMethod="Brightfield" EmissionWavelength="300.3" EmissionWavelengthUnit="nm" ExcitationWavelength="400.3" ExcitationWavelengthUnit="nm" Fluor="Fluor" ID="Channel:1:0" IlluminationType="Oblique" NDFilter="1.0" Name="Name" PinholeSize="0.5" PinholeSizeUnit="µm" PockelCellSetting="0" SamplesPerPixel="1"/>
+<Plane DeltaT="0.1" DeltaTUnit="s" ExposureTime="10.0" ExposureTimeUnit="s" PositionX="1.0" PositionXUnit="reference frame" PositionY="1.0" PositionYUnit="reference frame" PositionZ="1.0" PositionZUnit="reference frame" TheC="0" TheT="0" TheZ="0"/>
+<BinData xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06" Length="4" BigEndian="false">AQEBAQ==</BinData></Pixels></Image></OME>