From 84bb91d6f101b97b9bc01875dc1081a50fae15f5 Mon Sep 17 00:00:00 2001 From: Gabor Kovacs Date: Wed, 28 Jun 2023 00:14:33 -0700 Subject: [PATCH 1/2] Add support for ZarrKeyValueReader * Add ZarrKeyValueReaderBuilder to MultiscaleImage. * Update ZarrImageLoader and XmlIoZarrImageLoader to use ZarrKeyValueAccessBuilder. * Disable caching and merging - did not work. * Add "s3bucket" entry to ImageLoader in the xml. * Pom updates. * Add explicit anonymous authentication to S3 connection. --- pom.xml | 37 ++- src/main/java/bdv/img/omezarr/JsonTest.java | 2 - .../java/bdv/img/omezarr/MultiscaleImage.java | 227 ++++++++++++++++-- .../java/bdv/img/omezarr/Multiscales.java | 8 +- .../java/bdv/img/omezarr/S3ImgAccessTest.java | 24 ++ .../java/bdv/img/omezarr/S3TimingTest.java | 69 ++++++ .../bdv/img/omezarr/XmlIoZarrImageLoader.java | 43 +++- .../java/bdv/img/omezarr/ZarrImageLoader.java | 34 +-- 8 files changed, 379 insertions(+), 65 deletions(-) create mode 100644 src/main/java/bdv/img/omezarr/S3ImgAccessTest.java create mode 100644 src/main/java/bdv/img/omezarr/S3TimingTest.java diff --git a/pom.xml b/pom.xml index b0f21f6..0bebf1c 100644 --- a/pom.xml +++ b/pom.xml @@ -5,9 +5,7 @@ org.scijava pom-scijava - - 31.1.0 - + 36.0.0 org.bigdataviewer @@ -89,6 +87,11 @@ bsd_2 BigDataViewer developers. + + 3.0.0-SNAPSHOT + 3.2.1-SNAPSHOT + 0.0.9-SNAPSHOT + sign,deploy-to-scijava @@ -104,39 +107,51 @@ sc.fiji bigdataviewer-core - 10.4.6 sc.fiji bigdataviewer-vistools - 1.0.0-beta-32 net.imglib2 imglib2-cache - 1.0.0-beta-17 net.imglib2 imglib2 - 6.1.0 + + + + + + org.janelia.saalfeldlab + n5 org.janelia.saalfeldlab - n5-imglib2 - 5.0.0 + n5-aws-s3 org.janelia.saalfeldlab n5-zarr - 0.0.8 + + + org.janelia.saalfeldlab + n5-imglib2 + junit junit test - + + com.amazonaws + aws-java-sdk-s3 + 1.12.465 + + + diff --git a/src/main/java/bdv/img/omezarr/JsonTest.java b/src/main/java/bdv/img/omezarr/JsonTest.java index 5f120d1..2abd45e 100644 --- a/src/main/java/bdv/img/omezarr/JsonTest.java +++ b/src/main/java/bdv/img/omezarr/JsonTest.java @@ -30,9 +30,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import org.janelia.saalfeldlab.n5.GsonAttributesParser; import java.io.BufferedWriter; import java.io.IOException; diff --git a/src/main/java/bdv/img/omezarr/MultiscaleImage.java b/src/main/java/bdv/img/omezarr/MultiscaleImage.java index dff4238..65a35a3 100644 --- a/src/main/java/bdv/img/omezarr/MultiscaleImage.java +++ b/src/main/java/bdv/img/omezarr/MultiscaleImage.java @@ -28,11 +28,25 @@ */ package bdv.img.omezarr; -import bdv.img.cache.VolatileCachedCellImg; import bdv.cache.SharedQueue; import bdv.util.volatiles.VolatileTypeMatcher; import bdv.util.volatiles.VolatileViews; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.regions.DefaultAwsRegionProviderChain; +import com.amazonaws.regions.Regions; +import com.amazonaws.retry.PredefinedRetryPolicies; +import com.amazonaws.retry.RetryMode; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Arrays; import net.imglib2.RandomAccessibleInterval; import net.imglib2.Volatile; import net.imglib2.cache.img.CachedCellImg; @@ -51,13 +65,11 @@ import net.imglib2.util.Cast; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; +import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; import org.janelia.saalfeldlab.n5.zarr.N5ZarrReader; - -//import javax.annotation.Nullable; - -import java.io.IOException; -import java.util.ArrayList; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; import static bdv.img.omezarr.Multiscales.MULTI_SCALE_KEY; @@ -69,8 +81,158 @@ */ public class MultiscaleImage< T extends NativeType< T > & RealType< T >, V extends Volatile< T > & NativeType< V > & RealType< V > > { - private final String multiscalePath; + /** + * Helper class to store the base multiscale image path and other + * access credentials for creating readed instances either on local FS or from S3. + * + * Instances that are obtained by the copy-constructor after the credentials + * or the connection is set, will share the same AWS credential + * and connection instance. + * + */ + public static class ZarrKeyValueReaderBuilder { + private AmazonS3 s3Client; + private AWSCredentials s3Credentials = null; + private final boolean s3Mode; + private final String bucketName; + private String multiscalePath; + private String s3Region = null; + + /** + * Constructor for S3 reader. + * + * @param s3Client S3 access client. Instance is not copied. + * @param bucketName Bucket name for access. + * @param multiscalePath Base path within the bucket. + */ + public ZarrKeyValueReaderBuilder(final AmazonS3 s3Client, final String bucketName, + final String multiscalePath) { + this.multiscalePath = multiscalePath; + this.s3Client = s3Client; + this.bucketName = bucketName; + this.s3Mode = true; + } + + public ZarrKeyValueReaderBuilder(final String bucketName, + final String multiscalePath) + { + this(null, bucketName, multiscalePath); + } + public void setCredentials(AWSCredentials s3Credentials) + { + if (!s3Mode) + return; + this.s3Credentials = s3Credentials; + } + + public void setRegion(final String region) + { + if (!s3Mode) + return; + this.s3Region = region; + } + public void initS3Client() + { + if (!s3Mode) + return; + + if (s3Region==null) + s3Region = new DefaultAwsRegionProviderChain().getRegion(); + + if (s3Client==null) + { + final AWSStaticCredentialsProvider credentialsProvider; + if (s3Credentials==null) + { + try { + s3Credentials = new DefaultAWSCredentialsProviderChain().getCredentials(); + System.out.println( "Got credentials from default chain." ); + } + catch(final Exception e) { + System.out.println( "Could not load AWS credentials, falling back to anonymous." ); + } + } + credentialsProvider = new AWSStaticCredentialsProvider(s3Credentials == null ? new AnonymousAWSCredentials() : s3Credentials); + final ClientConfiguration s3Conf = new ClientConfiguration().withRetryPolicy(PredefinedRetryPolicies.getDefaultRetryPolicyWithCustomMaxRetries(32)); + s3Client = AmazonS3ClientBuilder.standard().withRegion(s3Region).withCredentials(credentialsProvider).withClientConfiguration(s3Conf).build(); + } + } + + /** + * Shallow copy constructor. + * + * Make shallow copy. S3 client instance will be shared. + * + * @param src Source instance. + */ + public ZarrKeyValueReaderBuilder(final ZarrKeyValueReaderBuilder src) + { + this.multiscalePath = src.multiscalePath; + this.s3Credentials = src.s3Credentials; + this.s3Client = src.s3Client; + this.bucketName = src.bucketName; + this.s3Mode = src.s3Mode; + } + + /** + * Constructor for filesystem access. + * @param multiscalePath Base path for zarr images. + */ + public ZarrKeyValueReaderBuilder(final String multiscalePath) { + this.multiscalePath = multiscalePath; + this.s3Client = null; + this.bucketName = null; + this.s3Mode = false; + } + + /** + * Build new reader instance. + * + * @return New N5ZarrReader instance for filesystem. New ZarrKeyValueReader instance for S3 access. + * @throws N5Exception New reader instance constructor exception propagates up. + */ + public ZarrKeyValueReader build() throws N5Exception { + if (s3Mode) { + initS3Client(); + final AmazonS3KeyValueAccess s3KeyValueAccess = new AmazonS3KeyValueAccess(s3Client, bucketName, false); + return new ZarrKeyValueReader(s3KeyValueAccess, multiscalePath, new GsonBuilder(), + false, false, true); + } else { + return new N5ZarrReader(multiscalePath); + } + } + + /** + * New reader builder for a sub-image. + * + * @param subPath Sub-path for sub-image in the zarr hierarchy. + * @return New shallow-copy builder instance pointing to base path/subpath. + */ + public ZarrKeyValueReaderBuilder getSubImage(final String subPath) { + final ZarrKeyValueReaderBuilder z = new ZarrKeyValueReaderBuilder(this); + if (s3Client == null) { + z.multiscalePath = Paths.get(z.multiscalePath, subPath).toString(); + } else { + z.multiscalePath = z.multiscalePath + "/" + subPath; + } + return z; + } + + /** + * @return Stored base path in this builder instance. + */ + public String getMultiscalePath() + { + return multiscalePath; + } + + /** + * @return Stored bucket name or null. + */ + public String getBucketName(){ return bucketName; } + } + private final ZarrKeyValueReaderBuilder zarrKeyValueReaderBuilder; private SharedQueue queue; private int numResolutions; @@ -98,11 +260,17 @@ public class MultiscaleImage< T extends NativeType< T > & RealType< T >, V exten /** * TODO */ + public MultiscaleImage( - final String multiscalePath) + final ZarrKeyValueReaderBuilder keyValueReaderBuilder) { - this.multiscalePath = multiscalePath; + this.zarrKeyValueReaderBuilder=keyValueReaderBuilder; } +// public MultiscaleImage( +// final String multiscalePath) +// { +// this(new ZarrKeyValueReaderBuilder(multiscalePath)); +// } /** * The delayed initialization of images is to have the shared queue set by ZarrImageLoader first @@ -112,7 +280,7 @@ private void initImages() if (imgs != null) return; try { - final N5ZarrReader n5ZarrReader = new N5ZarrReader(multiscalePath); +// final ZarrKeyValueReader zarrKeyValueReader = zarrKeyValueReaderBuilder.create(); // Initialize the images for all resolutions. // // TODO only on demand @@ -122,7 +290,7 @@ private void initImages() for ( int resolution = 0; resolution < numResolutions; ++resolution ) { - imgs[ resolution ] = N5Utils.openVolatile( n5ZarrReader, multiscales.getDatasets()[ resolution ].path ); + imgs[ resolution ] = N5Utils.openVolatile( zarrKeyValueReaderBuilder.build(), multiscales.getDatasets()[ resolution ].path ); if ( queue == null ) vimgs[ resolution ] = VolatileViews.wrapAsVolatile( imgs[ resolution ] ); @@ -142,11 +310,12 @@ private void initImages() private void init() { if ( multiscales != null ) return; - try (final N5ZarrReader n5ZarrReader = new N5ZarrReader( multiscalePath )) + zarrKeyValueReaderBuilder.initS3Client(); + try (final ZarrKeyValueReader zarrKeyValueReader = zarrKeyValueReaderBuilder.build()) { // Fetch metadata // - Multiscales[] multiscalesArray = n5ZarrReader.getAttribute( "", MULTI_SCALE_KEY, Multiscales[].class ); + Multiscales[] multiscalesArray = zarrKeyValueReader.getAttribute( "", MULTI_SCALE_KEY, Multiscales[].class ); // In principle the call above would be sufficient. // However since we need to support different @@ -156,7 +325,8 @@ private void init() // information. // TODO: could we do this by means of a JsonDeserializer? - final JsonArray multiscalesJsonArray = n5ZarrReader.getAttributes( "" ).get( MULTI_SCALE_KEY ).getAsJsonArray(); + final JsonArray multiscalesJsonArray; + multiscalesJsonArray = zarrKeyValueReader.getAttributes( "" ).getAsJsonObject().get( MULTI_SCALE_KEY ).getAsJsonArray(); for ( int i = 0; i < multiscalesArray.length; i++ ) { multiscalesArray[ i ].applyVersionFixes( multiscalesJsonArray.get( i ).getAsJsonObject() ); @@ -186,7 +356,7 @@ private void init() multiDataType = new DataType[numResolutions]; for (int resolution = numResolutions-1; resolution >= 0; --resolution) { - final DatasetAttributes attributes = n5ZarrReader.getDatasetAttributes(datasets[resolution].path); + final DatasetAttributes attributes = zarrKeyValueReader.getDatasetAttributes(datasets[resolution].path); // n5-zarr provides dimensions in the java order multiDimensions[resolution] = attributes.getDimensions(); multiDataType[resolution] = attributes.getDataType(); @@ -312,10 +482,27 @@ public int numDimensions() return dimensions.length; } - public static void main( String[] args ) - { - final String multiscalePath = "/Users/kgabor/data/davidf_sample_dataset/SmartSPIM_617052_sample.zarr"; - final MultiscaleImage< ?, ? > multiscaleImage = new MultiscaleImage<>( multiscalePath); - multiscaleImage.dimensions(); + public static void main( String[] args ) throws IOException { +// final String multiscalePath = "/home/gabor.kovacs/data/davidf_sample_dataset/SmartSPIM_617052_sample.zarr"; +// final MultiscaleImage< ?, ? > multiscaleImage = new MultiscaleImage<>(new ZarrKeyValueReaderBuilder(multiscalePath)); +// System.out.println(Arrays.toString(multiscaleImage.dimensions())); +// final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_WEST_2).build(); + final AWSStaticCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()); +// final AmazonS3 s3 = AmazonS3ClientBuilder.standard() +// .withCredentials(credentialsProvider) +// .withRegion(Regions.US_WEST_2) +// .build(); + final MultiscaleImage multiscaleImage = new MultiscaleImage<>( + new ZarrKeyValueReaderBuilder(AmazonS3ClientBuilder.standard() + .withCredentials(credentialsProvider) + .withRegion(Regions.US_WEST_2).build(), "aind-open-data", + "/exaSPIM_653431_2023-05-06_10-23-15/exaSPIM.zarr/tile_x_0000_y_0000_z_0000_ch_488.zarr/")); + System.out.println(Arrays.toString(multiscaleImage.dimensions())); + +// S3Object myobj = s3.getObject("aind-open-data","exaSPIM_653431_2023-05-06_10-23-15/exaSPIM.zarr/tile_x_0000_y_0000_z_0000_ch_488" +// +".zarr/.zattrs"); +// S3ObjectInputStream inputStream = myobj.getObjectContent(); +// String result = IOUtils.toString(inputStream, StandardCharsets.UTF_8); +// System.out.println(result); } } diff --git a/src/main/java/bdv/img/omezarr/Multiscales.java b/src/main/java/bdv/img/omezarr/Multiscales.java index 170e2c8..06304b6 100644 --- a/src/main/java/bdv/img/omezarr/Multiscales.java +++ b/src/main/java/bdv/img/omezarr/Multiscales.java @@ -31,7 +31,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; -import org.janelia.saalfeldlab.n5.GsonAttributesParser; +//import org.janelia.saalfeldlab.n5.GsonAttributesParser; import org.jcodings.util.Hash; import java.io.BufferedWriter; @@ -227,9 +227,9 @@ public static void main(String[] args ) throws IOException { M.datasets[0] = new Dataset(); m2.put(M.MULTI_SCALE_KEY, M); - GsonAttributesParser.insertAttributes(map, m2, gson); - final BufferedWriter w = new BufferedWriter(new OutputStreamWriter(System.out)); - GsonAttributesParser.writeAttributes(w, map, gson); +// GsonAttributesParser.insertAttributes(map, m2, gson); +// final BufferedWriter w = new BufferedWriter(new OutputStreamWriter(System.out)); +// GsonAttributesParser.writeAttributes(w, map, gson); } } diff --git a/src/main/java/bdv/img/omezarr/S3ImgAccessTest.java b/src/main/java/bdv/img/omezarr/S3ImgAccessTest.java new file mode 100644 index 0000000..92df126 --- /dev/null +++ b/src/main/java/bdv/img/omezarr/S3ImgAccessTest.java @@ -0,0 +1,24 @@ +package bdv.img.omezarr; + +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; + +import java.io.IOException; + +public class S3ImgAccessTest { + public static void main(String[] args) throws IOException { + final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_WEST_2).build(); + + System.out.println(""); + final AmazonS3KeyValueAccess mys3 = new AmazonS3KeyValueAccess(s3, "aind-open-data", false); + final ZarrKeyValueReader zreader = new ZarrKeyValueReader(mys3, + "/exaSPIM_653431_2023-05-06_10-23-15/exaSPIM.zarr/tile_x_0000_y_0000_z_0000_ch_488.zarr/", + new GsonBuilder(), true, true, true); + zreader.close(); + } + +} \ No newline at end of file diff --git a/src/main/java/bdv/img/omezarr/S3TimingTest.java b/src/main/java/bdv/img/omezarr/S3TimingTest.java new file mode 100644 index 0000000..a530c20 --- /dev/null +++ b/src/main/java/bdv/img/omezarr/S3TimingTest.java @@ -0,0 +1,69 @@ +package bdv.img.omezarr; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.stream.Collectors; + +import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; + +public class S3TimingTest { + public static void main(String[] args) throws IOException { + final String bucketName = "aind-open-data"; + final String path = "exaSPIM_653431_2023-05-06_10-23-15/exaSPIM.zarr/tile_x_0000_y_0000_z_0000_ch_488.zarr"; + long t0 = System.currentTimeMillis(); + +// final AWSStaticCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()); +// final AmazonS3 s3 = AmazonS3ClientBuilder.standard() +// .withCredentials(credentialsProvider) +// .withRegion(Regions.US_WEST_2) +// .build(); + + + // Try with leaving the default credentials provider chain in place + AWSCredentials credentials = null; + final AWSStaticCredentialsProvider credentialsProvider; + try { + credentials = new DefaultAWSCredentialsProviderChain().getCredentials(); + } + catch(final Exception e) { + System.out.println( "Could not load AWS credentials, falling back to anonymous." ); + } + credentialsProvider = new AWSStaticCredentialsProvider(credentials == null ? new AnonymousAWSCredentials() : credentials); + final AmazonS3 s3 = AmazonS3ClientBuilder.standard() + .withCredentials(credentialsProvider) + .withRegion(Regions.US_WEST_2) + .build(); + final AmazonS3KeyValueAccess kva = new AmazonS3KeyValueAccess(s3, bucketName, false); + + long t1 = System.currentTimeMillis(); + System.out.println("creating KeyValueAccess took: " + (t1 - t0) + " ms"); + + System.out.println("kva.isDirectory(path + \"/\") = " + kva.isDirectory(path + "/")); + long t2 = System.currentTimeMillis(); + System.out.println("took: " + (t2 - t1) + " ms"); + + System.out.println("kva.isFile(path + \"/.zattrs\") = " + kva.isFile(path + "/.zattrs")); + long t3 = System.currentTimeMillis(); + System.out.println("took: " + (t3 - t2) + " ms"); + + final InputStream inputStream = kva.lockForReading(path + "/.zattrs").newInputStream(); + String result = new BufferedReader(new InputStreamReader(inputStream)) + .lines().collect(Collectors.joining("\n")); + + long t4 = System.currentTimeMillis(); +// System.out.println("result = " + result); + System.out.println("reading .zattrs took: " + (t4 - t3) + " ms"); + + } +} + diff --git a/src/main/java/bdv/img/omezarr/XmlIoZarrImageLoader.java b/src/main/java/bdv/img/omezarr/XmlIoZarrImageLoader.java index 58138f3..00d93e1 100644 --- a/src/main/java/bdv/img/omezarr/XmlIoZarrImageLoader.java +++ b/src/main/java/bdv/img/omezarr/XmlIoZarrImageLoader.java @@ -29,13 +29,19 @@ package bdv.img.omezarr; -import bdv.BigDataViewer; import bdv.ViewerImgLoader; import bdv.ViewerSetupImgLoader; -import bdv.export.ProgressWriterConsole; import bdv.spimdata.SpimDataMinimal; import bdv.spimdata.XmlIoSpimDataMinimal; -import bdv.viewer.ViewerOptions; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import java.io.File; +import java.util.Map; +import java.util.TreeMap; import mpicbg.spim.data.SpimDataException; import mpicbg.spim.data.XmlHelpers; import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription; @@ -44,12 +50,6 @@ import mpicbg.spim.data.sequence.ViewId; import org.jdom2.Element; -import java.io.File; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -import bdv.img.omezarr.ZarrImageLoader; import static mpicbg.spim.data.XmlHelpers.loadPath; import static mpicbg.spim.data.XmlKeys.IMGLOADER_FORMAT_ATTRIBUTE_NAME; @@ -60,9 +60,17 @@ public class XmlIoZarrImageLoader implements XmlIoBasicImgLoader ze: imgLoader.getZgroups().entrySet()) { @@ -93,8 +101,17 @@ public ZarrImageLoader fromXml(final Element elem, final File basePath, final Ab final String path = c.getChild( "path" ).getText(); zgroups.put( new ViewId( timepointId,setupId ), path ); } - - return new ZarrImageLoader(zpath.getAbsolutePath(), zgroups, sequenceDescription); + final Element s3Bucket = elem.getChild("s3bucket"); + final MultiscaleImage.ZarrKeyValueReaderBuilder keyValueReaderBuilder; + if (s3Bucket==null) + { + keyValueReaderBuilder = new MultiscaleImage.ZarrKeyValueReaderBuilder(zpath.getAbsolutePath()); + } + else + { + keyValueReaderBuilder = new MultiscaleImage.ZarrKeyValueReaderBuilder(s3Bucket.getText(), zpath.toString()); + } + return new ZarrImageLoader(keyValueReaderBuilder, zgroups, sequenceDescription); } public static void main( String[] args ) throws SpimDataException @@ -108,7 +125,7 @@ public static void main( String[] args ) throws SpimDataException final ViewerSetupImgLoader setupImgLoader = imgLoader.getSetupImgLoader(0); int d = setupImgLoader.getImage(0).numDimensions(); setupImgLoader.getMipmapResolutions(); - BigDataViewer.open(spimData, "BigDataViewer Zarr Example", new ProgressWriterConsole(), ViewerOptions.options()); +// BigDataViewer.open(spimData, "BigDataViewer Zarr Example", new ProgressWriterConsole(), ViewerOptions.options()); System.out.println( "imgLoader = " + imgLoader ); System.out.println( "setupimgLoader = " + setupImgLoader ); } diff --git a/src/main/java/bdv/img/omezarr/ZarrImageLoader.java b/src/main/java/bdv/img/omezarr/ZarrImageLoader.java index 907082e..8f319d8 100644 --- a/src/main/java/bdv/img/omezarr/ZarrImageLoader.java +++ b/src/main/java/bdv/img/omezarr/ZarrImageLoader.java @@ -34,6 +34,16 @@ import bdv.cache.CacheControl; import bdv.cache.SharedQueue; import bdv.img.cache.VolatileGlobalCellCache; +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription; import mpicbg.spim.data.generic.sequence.BasicViewSetup; import mpicbg.spim.data.generic.sequence.ImgLoaderHint; @@ -49,14 +59,6 @@ import net.imglib2.type.NativeType; import net.imglib2.type.numeric.RealType; import net.imglib2.view.Views; -import bdv.img.omezarr.MultiscaleImage; -import bdv.img.omezarr.Multiscales; - -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.*; /** * Image loader for OME-NGFF images that are defined as views in the xml file. @@ -66,9 +68,11 @@ * Multiscale and MultiscaleImage keeps them as 5 dimensional.

*/ public class ZarrImageLoader implements ViewerImgLoader, MultiResolutionImgLoader, Closeable { - private final String zpath; +// private final String zpath; private final SortedMap zgroups; private final AbstractSequenceDescription seq; + private final MultiscaleImage.ZarrKeyValueReaderBuilder zarrKeyValueReaderBuilder; + private volatile boolean isOpen = false; @@ -80,10 +84,10 @@ public class ZarrImageLoader implements ViewerImgLoader, MultiResolutionImgLoade private SortedMap setupImgLoaders; - public ZarrImageLoader(final String zpath, final SortedMap zgroups, final AbstractSequenceDescription sequenceDescription) { - this.zpath = zpath; + public ZarrImageLoader(final MultiscaleImage.ZarrKeyValueReaderBuilder zarrKeyValueReaderBuilder, final SortedMap zgroups, final AbstractSequenceDescription sequenceDescription) { this.zgroups = zgroups; this.seq = sequenceDescription; + this.zarrKeyValueReaderBuilder=zarrKeyValueReaderBuilder; } void openZarr() { @@ -136,7 +140,7 @@ SetupImgLoader createSetupImgLoader(final int setupId) throws IOException if (firstVId == null) return null; - final MultiscaleImage mscImg = new MultiscaleImage<>(Paths.get(zpath, zgroups.get(firstVId)).toString()); + final MultiscaleImage mscImg = new MultiscaleImage<>(zarrKeyValueReaderBuilder.getSubImage(zgroups.get(firstVId))); return new SetupImgLoader(mscImg, firstVId, tpIds); } @@ -146,8 +150,8 @@ public CacheControl getCacheControl() { return cache; } - public File getBasePath() { - return new File(zpath); + public MultiscaleImage.ZarrKeyValueReaderBuilder getZarrKeyValueReaderBuilder() { + return zarrKeyValueReaderBuilder; } public SortedMap getZgroups() { @@ -181,7 +185,7 @@ public SetupImgLoader(final MultiscaleImage firstMscImg, final ViewId firs for (int tpId : tpIdSet) { // TODO validate that further timepoints have the same type tpMmultiscaleImages.put(tpId, new MultiscaleImage<>( - Paths.get(zpath, zgroups.get(new ViewId(tpId, setupId))).toString())); + zarrKeyValueReaderBuilder.getSubImage(zgroups.get(new ViewId(tpId, setupId))))); } calculateMipmapTransforms(); From 92b4c0f5eb3aa80e3c72b362d9e428a874e85bb6 Mon Sep 17 00:00:00 2001 From: Gabor Kovacs Date: Tue, 10 Oct 2023 22:08:26 +0200 Subject: [PATCH 2/2] Update n5 package versions. --- pom.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 0bebf1c..30575b8 100644 --- a/pom.xml +++ b/pom.xml @@ -87,10 +87,9 @@ bsd_2 BigDataViewer developers. - - 3.0.0-SNAPSHOT - 3.2.1-SNAPSHOT - 0.0.9-SNAPSHOT + 3.0.2 + 4.0.1 + 1.0.1 sign,deploy-to-scijava