Merge pull request #221 from melissalinkert/progress-api-part2

Update progress listener API to provide tile counts at each start notification

src/main/java/com/glencoesoftware/bioformats2raw/Converter.java

@@ -209,6 +209,7 @@ public class Converter implements Callable<Integer> {
   private volatile Path outputPath;
 
   private IProgressListener progressListener;
+  private Map<Integer, int[]> tileCounts = new HashMap<Integer, int[]>();
 
   // Option setters
 
@@ -1337,6 +1338,17 @@ public void convert()
         root.writeAttributes(attributes);
       }
 
+      // pre-calculate resolution and tile counts
+      long totalTiles = 0;
+      for (Integer index : seriesList) {
+        int[] counts = calculateTileCounts(index);
+        tileCounts.put(index, counts);
+        for (int count : counts) {
+          totalTiles += count;
+        }
+      }
+      getProgressListener().notifyStart(seriesList.size(), totalTiles);
+
       for (Integer index : seriesList) {
         try {
           write(index);
@@ -1375,6 +1387,105 @@ public void convert()
     }
   }
 
+  /**
+   * Pre-calculate the number of resolutions and tiles for the
+   * given series index. This is useful for accurate progress reporting
+   * and failing faster for incompatible pixel/downsampling combinations.
+   *
+   * @param series series index
+   * @return array of tile counts; the array length is the number of resolutions
+   */
+  private int[] calculateTileCounts(int series)
+    throws FormatException, IOException, InterruptedException,
+           EnumerationException
+  {
+    readers.forEach((reader) -> {
+      reader.setSeries(series);
+    });
+
+    IFormatReader workingReader = readers.take();
+    int resolutions = 1;
+    int sizeX;
+    int sizeY;
+    int sizeZ;
+    int imageCount;
+    try {
+      // calculate a reasonable pyramid depth if not specified as an argument
+      sizeX = workingReader.getSizeX();
+      sizeY = workingReader.getSizeY();
+      if (pyramidResolutions == null) {
+        if (workingReader.getResolutionCount() > 1
+            && reuseExistingResolutions)
+        {
+          resolutions = workingReader.getResolutionCount();
+        }
+        else {
+          resolutions = calculateResolutions(sizeX, sizeY);
+        }
+      }
+      else {
+        resolutions = pyramidResolutions;
+
+        // check to make sure too many resolutions aren't being used
+        if ((int) (sizeX / Math.pow(PYRAMID_SCALE, resolutions)) == 0 ||
+          (int) (sizeY / Math.pow(PYRAMID_SCALE, resolutions)) == 0)
+        {
+          resolutions = calculateResolutions(sizeX, sizeY);
+          LOGGER.warn("Too many resolutions specified; reducing to {}",
+            resolutions);
+        }
+      }
+      LOGGER.info("Using {} pyramid resolutions", resolutions);
+      sizeZ = workingReader.getSizeZ();
+      imageCount = workingReader.getImageCount();
+      pixelType = workingReader.getPixelType();
+    }
+    finally {
+      readers.put(workingReader);
+    }
+
+    int[] resTileCounts = new int[resolutions];
+
+    if ((pixelType == FormatTools.INT8 || pixelType == FormatTools.INT32) &&
+      getDownsampling() != Downsampling.SIMPLE && resolutions > 0)
+    {
+      String type = FormatTools.getPixelTypeString(pixelType);
+      throw new UnsupportedOperationException(
+        "OpenCV does not support downsampling " + type + " data. " +
+        "See https://github.com/opencv/opencv/issues/7862");
+    }
+
+    for (int resCounter=0; resCounter<resolutions; resCounter++) {
+      final int resolution = resCounter;
+      int scale = (int) Math.pow(PYRAMID_SCALE, resolution);
+      int scaledWidth = sizeX / scale;
+      int scaledHeight = sizeY / scale;
+      int scaledDepth = sizeZ;
+
+      workingReader = readers.take();
+      try {
+        if (workingReader.getResolutionCount() > 1
+            && reuseExistingResolutions)
+        {
+          workingReader.setResolution(resCounter);
+          scaledWidth = workingReader.getSizeX();
+          scaledHeight = workingReader.getSizeY();
+          scaledDepth = workingReader.getSizeZ();
+        }
+      }
+      finally {
+        readers.put(workingReader);
+      }
+
+      resTileCounts[resCounter] =
+          (int) Math.ceil((double) scaledWidth / tileWidth)
+          * (int) Math.ceil((double) scaledHeight / tileHeight)
+          * (int) Math.ceil((double) scaledDepth / chunkDepth)
+          * (imageCount / sizeZ);
+    }
+    return resTileCounts;
+  }
+
   /**
    * Convert the data specified by the given initialized reader to
    * an intermediate form.
@@ -1835,9 +1946,15 @@ public void saveResolutions(int series)
     throws FormatException, IOException, InterruptedException,
            EnumerationException
   {
-    getProgressListener().notifySeriesStart(series);
+    int[] resTileCounts = tileCounts.get(series);
+    int resolutions = resTileCounts.length;
+    int seriesTiles = 0;
+    for (int t : resTileCounts) {
+      seriesTiles += t;
+    }
+    getProgressListener().notifySeriesStart(series, resolutions, seriesTiles);
+
     IFormatReader workingReader = readers.take();
-    int resolutions = 1;
     int sizeX;
     int sizeY;
     int sizeZ;
@@ -1849,29 +1966,6 @@ public void saveResolutions(int series)
       // calculate a reasonable pyramid depth if not specified as an argument
       sizeX = workingReader.getSizeX();
       sizeY = workingReader.getSizeY();
-      if (pyramidResolutions == null) {
-        if (workingReader.getResolutionCount() > 1
-            && reuseExistingResolutions)
-        {
-          resolutions = workingReader.getResolutionCount();
-        }
-        else {
-          resolutions = calculateResolutions(sizeX, sizeY);
-        }
-      }
-      else {
-        resolutions = pyramidResolutions;
-
-        // check to make sure too many resolutions aren't being used
-        if ((int) (sizeX / Math.pow(PYRAMID_SCALE, resolutions)) == 0 ||
-          (int) (sizeY / Math.pow(PYRAMID_SCALE, resolutions)) == 0)
-        {
-          resolutions = calculateResolutions(sizeX, sizeY);
-          LOGGER.warn("Too many resolutions specified; reducing to {}",
-            resolutions);
-        }
-      }
-      LOGGER.info("Using {} pyramid resolutions", resolutions);
       sizeZ = workingReader.getSizeZ();
       sizeT = workingReader.getSizeT();
       sizeC = workingReader.getSizeC();
@@ -1883,15 +1977,6 @@ public void saveResolutions(int series)
       readers.put(workingReader);
     }
 
-    if ((pixelType == FormatTools.INT8 || pixelType == FormatTools.INT32) &&
-      getDownsampling() != Downsampling.SIMPLE && resolutions > 0)
-    {
-      String type = FormatTools.getPixelTypeString(pixelType);
-      throw new UnsupportedOperationException(
-        "OpenCV does not support downsampling " + type + " data. " +
-        "See https://github.com/opencv/opencv/issues/7862");
-    }
-
     LOGGER.info(
       "Preparing to write pyramid sizeX {} (tileWidth: {}) " +
       "sizeY {} (tileWidth: {}) sizeZ {} (tileDepth: {}) imageCount {}",
@@ -1958,10 +2043,7 @@ public void saveResolutions(int series)
       ZarrArray.create(getRootPath().resolve(resolutionString), arrayParams);
 
       nTile = new AtomicInteger(0);
-      tileCount = (int) Math.ceil((double) scaledWidth / tileWidth)
-          * (int) Math.ceil((double) scaledHeight / tileHeight)
-          * (int) Math.ceil((double) scaledDepth / chunkDepth)
-          * (imageCount / sizeZ);
+      tileCount = resTileCounts[resolution];
 
       List<CompletableFuture<Void>> futures =
         new ArrayList<CompletableFuture<Void>>();

src/main/java/com/glencoesoftware/bioformats2raw/IProgressListener.java

@@ -11,12 +11,23 @@
 
 public interface IProgressListener extends EventListener {
 
+  /**
+   * Indicates the total number of chunks in this conversion operation.
+   * Includes all resolutions in all series.
+   *
+   * @param seriesCount total number of series
+   * @param chunkCount total number of chunks
+   */
+  void notifyStart(int seriesCount, long chunkCount);
+
   /**
    * Indicates the beginning of processing a particular series.
    *
    * @param series the series index being processed
+   * @param resolutionCount total number of resolutions in this series
+   * @param chunkCount total number of chunks for all resolutions in this series
    */
-  void notifySeriesStart(int series);
+  void notifySeriesStart(int series, int resolutionCount, int chunkCount);
 
   /**
    * Indicates the end of processing a particular series.
@@ -29,9 +40,9 @@ public interface IProgressListener extends EventListener {
    * Indicates the beginning of processing a particular resolution.
    *
    * @param resolution the resolution index being processed
-   * @param tileCount the total number of tiles in this resolution
+   * @param chunkCount the total number of chunks in this resolution
    */
-  void notifyResolutionStart(int resolution, int tileCount);
+  void notifyResolutionStart(int resolution, int chunkCount);
 
   /**
    * Indicates the end of processing a particular resolution.

src/main/java/com/glencoesoftware/bioformats2raw/NoOpProgressListener.java

@@ -10,15 +10,21 @@
 public class NoOpProgressListener implements IProgressListener {
 
   @Override
-  public void notifySeriesStart(int series) {
+  public void notifyStart(int seriesCount, long chunkCount) {
+  }
+
+  @Override
+  public void notifySeriesStart(int series, int resolutionCount,
+    int chunkCount)
+  {
   }
 
   @Override
   public void notifySeriesEnd(int series) {
   }
 
   @Override
-  public void notifyResolutionStart(int resolution, int tileCount) {
+  public void notifyResolutionStart(int resolution, int chunkCount) {
   }
 
   @Override

src/main/java/com/glencoesoftware/bioformats2raw/ProgressBarListener.java

@@ -32,20 +32,27 @@ public ProgressBarListener(String level) {
     logLevel = level;
   }
 
+  @Override
+  public void notifyStart(int seriesCount, long chunkCount) {
+    // intentional no-op
+  }
 
   @Override
-  public void notifySeriesStart(int series) {
+  public void notifySeriesStart(int series, int resolutionCount,
+    int chunkCount)
+  {
     currentSeries = series;
   }
 
   @Override
   public void notifySeriesEnd(int series) {
+    // intentional no-op
   }
 
   @Override
-  public void notifyResolutionStart(int resolution, int tileCount) {
+  public void notifyResolutionStart(int resolution, int chunkCount) {
     ProgressBarBuilder builder = new ProgressBarBuilder()
-      .setInitialMax(tileCount)
+      .setInitialMax(chunkCount)
       .setTaskName(String.format("[%d/%d]", currentSeries, resolution));
 
     if (!(logLevel.equals("OFF") ||

src/test/java/com/glencoesoftware/bioformats2raw/test/TestProgressListener.java

@@ -18,9 +18,17 @@ public class TestProgressListener implements IProgressListener {
   private int startedTiles = 0;
   private int completedTiles = 0;
   private int expectedTileCount = 0;
+  private int seriesTiles = 0;
+  private long totalTiles = 0;
 
   @Override
-  public void notifySeriesStart(int series) {
+  public void notifyStart(int seriesCount, long tileCount) {
+    totalTiles = tileCount;
+  }
+
+  @Override
+  public void notifySeriesStart(int series, int res, int tiles) {
+    seriesTiles = tiles;
   }
 
   @Override
@@ -60,8 +68,22 @@ public void notifyResolutionEnd(int resolution) {
    *
    * @return an array with one element per resolution
    */
-  public Integer[] getTileCounts() {
+  public Integer[] getChunkCounts() {
     return finishedResolutions.toArray(new Integer[finishedResolutions.size()]);
   }
 
+  /**
+   * @return the reported number of tiles for the most recent series
+   */
+  public int getSeriesChunkCount() {
+    return seriesTiles;
+  }
+
+  /**
+   * @return the reported number of total tiles for the conversion
+   */
+  public long getTotalChunkCount() {
+    return totalTiles;
+  }
+
 }

src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java

@@ -708,9 +708,15 @@ public void testProgressListener() throws Exception {
       throw new RuntimeException(t);
     }
 
-    Integer[] expectedTileCounts = new Integer[] {320, 80, 20, 5, 5, 5};
-    Integer[] tileCounts = listener.getTileCounts();
-    assertArrayEquals(expectedTileCounts, tileCounts);
+    Integer[] expectedChunkCounts = new Integer[] {320, 80, 20, 5, 5, 5};
+    Integer[] chunkCounts = listener.getChunkCounts();
+    assertArrayEquals(expectedChunkCounts, chunkCounts);
+    long totalChunkCount = 0;
+    for (Integer t : expectedChunkCounts) {
+      totalChunkCount += t;
+    }
+    assertEquals(totalChunkCount, listener.getTotalChunkCount());
+    assertEquals(totalChunkCount, listener.getSeriesChunkCount());
   }
 
   private int bytesPerPixel(DataType dataType) {