diff --git a/.gitignore b/.gitignore index 95ef79ac..94ed2dbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target* __pycache__/ +node_modules/ \ No newline at end of file diff --git a/Makefile b/Makefile index 16f41da4..be2fc6ff 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,11 @@ z5py: data/reference_image.png zarr: data/reference_image.png python generate_data/generate_zarr.py +.PHONY: js +js: data/reference_image.png + bash generate_data/js/generate_data.sh + .PHONY: data -data: n5java pyn5 z5py zarr +data: n5java pyn5 z5py zarr js .PHONY: test diff --git a/environment.yml b/environment.yml index 60309009..c7b27b05 100644 --- a/environment.yml +++ b/environment.yml @@ -5,6 +5,7 @@ dependencies: - openjdk - maven - make + - nodejs - z5py >= 2.0.8 - python == 3.7.9 - scikit-image diff --git a/generate_data/js/generate_data.sh b/generate_data/js/generate_data.sh new file mode 100755 index 00000000..c70e51f0 --- /dev/null +++ b/generate_data/js/generate_data.sh @@ -0,0 +1,6 @@ +# cd to this directory +# https://stackoverflow.com/a/6393573/2700168 +cd "${0%/*}" + +npm install +npm start diff --git a/generate_data/js/package-lock.json b/generate_data/js/package-lock.json new file mode 100644 index 00000000..f95b52a6 --- /dev/null +++ b/generate_data/js/package-lock.json @@ -0,0 +1,139 @@ +{ + "name": "js", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "dependencies": { + "pngjs": "^6.0.0", + "zarr": "^0.4.0" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, + "node_modules/numcodecs": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.1.1.tgz", + "integrity": "sha512-UjKulZ6GIFKLdBIczEbsoXNZQmiHafpoIdo39YcdecHVGyMKh0+azsfHTrybXm5RZwepqLZv24mkjqGdZGm24Q==", + "dependencies": { + "pako": "^1.0.11" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.2.0.tgz", + "integrity": "sha512-B2LXNONcyn/G6uz2UBFsGjmSa0e/br3jznlzhEyCXg56c7VhEpiT2pZxGOfv32Q3FSyugAdys9KGpsv3kV+Sbg==", + "dependencies": { + "eventemitter3": "^4.0.0", + "p-timeout": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "node_modules/pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/zarr": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/zarr/-/zarr-0.4.0.tgz", + "integrity": "sha512-zvxdX3aRWxjy6H3OtA7R05NNZvRKxn/7bkNJhUsVKKbNoJ3DBqYERQfzI4WfAV1OTcclqvlYwkQ7DWsGJA5QEw==", + "dependencies": { + "numcodecs": "^0.1.0", + "p-queue": "6.2.0" + }, + "engines": { + "node": ">=8.0.0" + } + } + }, + "dependencies": { + "eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, + "numcodecs": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.1.1.tgz", + "integrity": "sha512-UjKulZ6GIFKLdBIczEbsoXNZQmiHafpoIdo39YcdecHVGyMKh0+azsfHTrybXm5RZwepqLZv24mkjqGdZGm24Q==", + "requires": { + "pako": "^1.0.11" + } + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, + "p-queue": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.2.0.tgz", + "integrity": "sha512-B2LXNONcyn/G6uz2UBFsGjmSa0e/br3jznlzhEyCXg56c7VhEpiT2pZxGOfv32Q3FSyugAdys9KGpsv3kV+Sbg==", + "requires": { + "eventemitter3": "^4.0.0", + "p-timeout": "^3.1.0" + } + }, + "p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "requires": { + "p-finally": "^1.0.0" + } + }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==" + }, + "zarr": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/zarr/-/zarr-0.4.0.tgz", + "integrity": "sha512-zvxdX3aRWxjy6H3OtA7R05NNZvRKxn/7bkNJhUsVKKbNoJ3DBqYERQfzI4WfAV1OTcclqvlYwkQ7DWsGJA5QEw==", + "requires": { + "numcodecs": "^0.1.0", + "p-queue": "6.2.0" + } + } + } +} diff --git a/generate_data/js/package.json b/generate_data/js/package.json new file mode 100644 index 00000000..60d38ffc --- /dev/null +++ b/generate_data/js/package.json @@ -0,0 +1,10 @@ +{ + "type": "module", + "scripts": { + "start": "node src/index.js" + }, + "dependencies": { + "pngjs": "^6.0.0", + "zarr": "^0.4.0" + } +} diff --git a/generate_data/js/src/fsstore.js b/generate_data/js/src/fsstore.js new file mode 100644 index 00000000..f44974d8 --- /dev/null +++ b/generate_data/js/src/fsstore.js @@ -0,0 +1,37 @@ +import path from "path"; +import { promises as fsp } from "fs"; +import fs from "fs"; + +import { KeyError } from "zarr"; + +class FSStore { + constructor(root) { + this.root = root; + if (!fs.existsSync(root)) { + fs.mkdirSync(root, { recursive: true }); + } + } + + getItem(key) { + const fp = path.join(this.root, key); + return fsp.readFile(fp, null).catch((err) => { + if (err.code === "ENOENT") { + throw new KeyError(key); + } + throw err; + }); + } + + async setItem(key, value) { + const fp = path.join(this.root, key); + await fsp.mkdir(path.dirname(fp), { recursive: true }); + await fsp.writeFile(fp, Buffer.from(value), null); + } + + containsItem(key) { + const fp = path.join(this.root, key); + return fs.existsSync(fp); + } +} + +export default FSStore; diff --git a/generate_data/js/src/index.js b/generate_data/js/src/index.js new file mode 100644 index 00000000..455e0099 --- /dev/null +++ b/generate_data/js/src/index.js @@ -0,0 +1,53 @@ +import fs from "fs"; +import p from "path"; +import { PNG } from "pngjs"; + +import { openGroup, NestedArray, slice } from "zarr"; +import FSStore from "./fsstore.js"; + +const CHUNKS = [100, 100, 1]; +const STR_TO_COMPRESSOR = { + gzip: { id: "gzip", level: 1 }, + blosc: { id: "blosc", cname: "lz4", clevel: 5, blocksize: 0, shuffle: 1 }, + zlib: { id: "zlib", level: 1 }, +}; + +// Simple convenience method to init the root for an empty store. +async function open(path) { + const store = new FSStore(path); + const text = JSON.stringify({ zarr_format: 2 }); + await store.setItem(".zgroup", Buffer.from(text)); + return openGroup(store); +} + +function imread(path) { + const buf = fs.readFileSync(path); + const { data, height, width } = PNG.sync.read(buf, { colorType: 2 }); + const arr = new NestedArray(new Uint8Array(data), [height, width, 4]); + return arr.get([null, null, slice(3)]); // drop alpha channel +} + +function getName(config) { + if (config === null) return "raw"; + if (config.cname) return `${config.id}/${config.cname}`; + return config.id; +} + +async function generateZarrFormat(codecIds = ["gzip", "blosc", "zlib", null]) { + const path = p.join("..", "..", "data", "js.zr"); + const img = imread(p.join("..", "..", "data", "reference_image.png")); + + fs.rmSync(path, { recursive: true, force: true }); + const grp = await open(path); + for (const id of codecIds) { + const config = id ? STR_TO_COMPRESSOR[id] : null; + const name = getName(config); + grp.createDataset(name, undefined, img, { + compressor: config, + chunks: CHUNKS, + fillValue: 0, + }); + } +} + +generateZarrFormat();