diff --git a/build.gradle b/build.gradle index eb9e462..f8e2bda 100644 --- a/build.gradle +++ b/build.gradle @@ -59,6 +59,8 @@ dependencies { test { useJUnit() + + maxHeapSize = "2g" } jar { diff --git a/src/main/java/com/glencoesoftware/zarr/Convert.java b/src/main/java/com/glencoesoftware/zarr/Convert.java index 966ff40..66ca616 100644 --- a/src/main/java/com/glencoesoftware/zarr/Convert.java +++ b/src/main/java/com/glencoesoftware/zarr/Convert.java @@ -37,6 +37,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.Callable; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -44,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ch.qos.logback.classic.Level; import picocli.CommandLine; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @@ -61,6 +63,7 @@ public class Convert implements Callable { private String inputLocation; private String outputLocation; + private String logLevel = "INFO"; private boolean writeV2; private ShardConfiguration shardConfig; @@ -90,6 +93,26 @@ public void setOutput(String output) { outputLocation = output; } + /** + * Set the slf4j logging level. Defaults to "INFO". + * + * @param level logging level + */ + @Option( + names = {"--log-level", "--debug"}, + arity = "0..1", + description = "Change logging level; valid values are " + + "OFF, ERROR, WARN, INFO, DEBUG, TRACE and ALL. " + + "(default: ${DEFAULT-VALUE})", + defaultValue = "INFO", + fallbackValue = "DEBUG" + ) + public void setLogLevel(String level) { + if (level != null) { + logLevel = level; + } + } + @Option( names = "--write-v2", description = "Read v3, write v2", @@ -134,6 +157,10 @@ public void setCompression(String[] compression) { @Override public Integer call() throws Exception { + ch.qos.logback.classic.Logger root = (ch.qos.logback.classic.Logger) + LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + root.setLevel(Level.toLevel(logLevel)); + if (writeV2) { convertToV2(); } @@ -151,6 +178,7 @@ public void convertToV3() throws Exception { Path inputPath = Paths.get(inputLocation); // get the root-level attributes + LOGGER.debug("opening v2 root group: {}", inputPath); ZarrGroup reader = ZarrGroup.open(inputPath); Map attributes = reader.getAttributes(); @@ -163,6 +191,7 @@ public void convertToV3() throws Exception { // but this doesn't seem to actually create the group // separating the group creation and attribute writing into // two calls seems to work correctly + LOGGER.debug("opening v3 root group: {}", outputLocation); FilesystemStore outputStore = new FilesystemStore(outputLocation); Group outputRootGroup = Group.create(outputStore.resolve()); outputRootGroup.setAttributes(attributes); @@ -175,9 +204,11 @@ public void convertToV3() throws Exception { for (String seriesGroupKey : groupKeys) { if (seriesGroupKey.indexOf("/") > 0) { + LOGGER.debug("skipping v2 group key: {}", seriesGroupKey); continue; } Path seriesPath = inputPath.resolve(seriesGroupKey); + LOGGER.debug("opening v2 group: {}", seriesPath); ZarrGroup seriesGroup = ZarrGroup.open(seriesPath); LOGGER.info("opened {}", seriesPath); @@ -190,13 +221,16 @@ public void convertToV3() throws Exception { Set columnKeys = seriesGroup.getGroupKeys(); // "pass through" if this is not HCS if (columnKeys.size() == 0) { + LOGGER.debug("no column group keys (likely not HCS)"); columnKeys.add(""); } for (String columnKey : columnKeys) { if (columnKey.indexOf("/") > 0) { + LOGGER.debug("skipping v2 column group key: {}", columnKey); continue; } Path columnPath = columnKey.isEmpty() ? seriesPath : seriesPath.resolve(columnKey); + LOGGER.debug("opening v2 group: {}", columnPath); ZarrGroup column = ZarrGroup.open(columnPath); if (!columnKey.isEmpty()) { @@ -208,14 +242,15 @@ public void convertToV3() throws Exception { Set fieldKeys = column.getGroupKeys(); // "pass through" if this is not HCS if (fieldKeys.size() == 0) { + LOGGER.debug("no field group keys"); fieldKeys.add(""); } for (String fieldKey : fieldKeys) { Path fieldPath = fieldKey.isEmpty() ? columnPath : columnPath.resolve(fieldKey); + LOGGER.debug("opening v2 field group: {}", fieldPath); ZarrGroup field = ZarrGroup.open(fieldPath); - Map fieldAttributes = field.getAttributes(); if (!fieldKey.isEmpty()) { Group outputFieldGroup = Group.create(outputStore.resolve(seriesGroupKey, columnKey, fieldKey)); @@ -239,12 +274,16 @@ public void convertToV3() throws Exception { for (int res=0; res builder) .build() ); - for (int t=0; t