From fe91b7f1426e70913267f5ebeb8ac2c8a586b852 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 24 Apr 2019 18:50:24 +0100 Subject: [PATCH 1/4] feat: switch to async await --- package.json | 15 +- src/builder/builder.js | 7 +- src/builder/reduce.js | 6 +- src/importer/dir-flat.js | 10 +- src/importer/dir-sharded.js | 10 +- src/importer/index.js | 1 + src/importer/tree-builder.js | 9 +- src/index.js | 25 +- src/utils/persist.js | 23 +- test/benchmark.spec.js | 36 +- test/builder-balanced.spec.js | 117 ++-- test/builder-dir-sharding.spec.js | 536 +++++++---------- test/builder-flat.spec.js | 43 +- test/builder-only-hash.spec.js | 50 +- test/builder-trickle-dag.spec.js | 802 ++++++++++++------------- test/builder.spec.js | 116 ++-- test/chunker-fixed-size.spec.js | 122 ++-- test/chunker-rabin-browser.spec.js | 35 +- test/chunker-rabin.spec.js | 84 ++- test/hash-parity-with-go-ipfs.spec.js | 31 +- test/helpers/collect-leaf-cids.js | 47 +- test/helpers/stream-to-array.js | 0 test/import-export-nested-dir.spec.js | 143 ++--- test/import-export.spec.js | 41 +- test/importer-flush.spec.js | 2 +- test/importer.spec.js | 811 +++++++++++--------------- test/with-dag-api.spec.js | 436 -------------- 27 files changed, 1386 insertions(+), 2172 deletions(-) create mode 100644 test/helpers/stream-to-array.js delete mode 100644 test/with-dag-api.spec.js diff --git a/package.json b/package.json index 8cf6b9e..ffd5ea6 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "release": "aegir release", "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", - "coverage": "aegir coverage" + "coverage": "aegir coverage", + "dep-check": "aegir dep-check" }, "repository": { "type": "git", @@ -38,28 +39,33 @@ "homepage": "https://github.com/ipfs/js-ipfs-unixfs-importer#readme", "devDependencies": { "aegir": "^18.0.2", + "async-iterator-all": "0.0.2", + "async-iterator-buffer-stream": "~0.0.1", + "async-iterator-first": "0.0.2", + "async-iterator-last": "0.0.2", "chai": "^4.2.0", "cids": "~0.5.5", "detect-node": "^2.0.4", "dirty-chai": "^2.0.1", "ipfs-unixfs-exporter": "~0.36.1", - "ipld": "~0.21.1", + "ipld": "~0.22.0", "ipld-in-memory": "^2.0.0", "multihashes": "~0.4.14", - "pull-buffer-stream": "^1.0.1", "pull-generate": "^2.2.0", "pull-traverse": "^1.0.3", "sinon": "^7.1.0" }, "dependencies": { "async": "^2.6.1", - "async-iterator-to-pull-stream": "^1.1.0", + "async-iterator-to-pull-stream": "^1.3.0", "bl": "^3.0.0", "deep-extend": "~0.6.0", + "err-code": "^1.1.2", "hamt-sharding": "~0.0.2", "ipfs-unixfs": "~0.1.16", "ipld-dag-pb": "~0.15.2", "left-pad": "^1.3.0", + "multicodec": "~0.5.1", "multihashing-async": "~0.5.1", "pull-batch": "^1.0.0", "pull-pair": "^1.1.0", @@ -67,6 +73,7 @@ "pull-pause": "0.0.2", "pull-pushable": "^2.2.0", "pull-stream": "^3.6.9", + "pull-stream-to-async-iterator": "^1.0.1", "pull-through": "^1.0.18", "pull-write": "^1.1.4", "stream-to-pull-stream": "^1.7.2" diff --git a/src/builder/builder.js b/src/builder/builder.js index f6d1766..c97ba9d 100644 --- a/src/builder/builder.js +++ b/src/builder/builder.js @@ -15,6 +15,7 @@ const reduce = require('./reduce') const { DAGNode } = require('ipld-dag-pb') +const errCode = require('err-code') const defaultOptions = { chunkerOptions: { @@ -78,7 +79,7 @@ module.exports = function builder (createChunker, ipld, createReducer, _options) callback(null, { path: item.path, - multihash: result.cid.buffer, + cid: result.cid, size: result.node.size }) }) @@ -90,7 +91,7 @@ module.exports = function builder (createChunker, ipld, createReducer, _options) } if (typeof file.content !== 'function') { - return callback(new Error('invalid content')) + return callback(errCode(new Error('invalid content'), 'EINVALIDCONTENT')) } const reducer = createReducer(reduce(file, ipld, options), options) @@ -146,7 +147,7 @@ module.exports = function builder (createChunker, ipld, createReducer, _options) size: leaf.size, leafSize: leaf.leafSize, data: results.node, - multihash: results.cid.buffer, + cid: results.cid, path: leaf.path, name: '' }) diff --git a/src/builder/reduce.js b/src/builder/reduce.js index 2979bab..99b9d55 100644 --- a/src/builder/reduce.js +++ b/src/builder/reduce.js @@ -16,7 +16,7 @@ module.exports = function reduce (file, ipld, options) { return callback(null, { size: leaf.size, leafSize: leaf.leafSize, - multihash: leaf.multihash, + cid: leaf.cid, path: file.path, name: leaf.name }) @@ -28,7 +28,7 @@ module.exports = function reduce (file, ipld, options) { const links = leaves.map((leaf) => { f.addBlockSize(leaf.leafSize) - return new DAGLink(leaf.name, leaf.size, leaf.multihash) + return new DAGLink(leaf.name, leaf.size, leaf.cid) }) waterfall([ @@ -42,7 +42,7 @@ module.exports = function reduce (file, ipld, options) { callback(null, { size: result.node.size, leafSize: f.fileSize(), - multihash: result.cid.buffer, + cid: result.cid, path: file.path, name: '' }) diff --git a/src/importer/dir-flat.js b/src/importer/dir-flat.js index 549662f..22efc37 100644 --- a/src/importer/dir-flat.js +++ b/src/importer/dir-flat.js @@ -16,7 +16,7 @@ class DirFlat extends Dir { } put (name, value, callback) { - this.multihash = undefined + this.cid = undefined this.size = undefined this._children[name] = value process.nextTick(callback) @@ -52,7 +52,7 @@ class DirFlat extends Dir { const links = Object.keys(this._children) .map((key) => { const child = this._children[key] - return new DAGLink(key, child.size, child.multihash) + return new DAGLink(key, child.size, child.cid) }) const dir = new UnixFS('directory') @@ -62,12 +62,12 @@ class DirFlat extends Dir { (callback) => DAGNode.create(dir.marshal(), links, callback), (node, callback) => persist(node, ipld, this._options, callback), ({ cid, node }, callback) => { - this.multihash = cid.buffer + this.cid = cid this.size = node.size const pushable = { path: path, - multihash: cid.buffer, - size: node.size + size: node.size, + cid: cid } source.push(pushable) callback(null, node) diff --git a/src/importer/dir-sharded.js b/src/importer/dir-sharded.js index a209eb3..32a7936 100644 --- a/src/importer/dir-sharded.js +++ b/src/importer/dir-sharded.js @@ -52,6 +52,10 @@ class DirSharded extends Dir { } async put (name, value, callback) { + if (!callback) { + console.info('wut') + } + try { await this._bucket.put(name, value) @@ -100,7 +104,7 @@ class DirSharded extends Dir { if (err) { return callback(err) } else { - this.multihash = results.cid.buffer + this.cid = results.cid this.size = results.node.size } @@ -157,7 +161,7 @@ function flush (options, bucket, path, ipld, source, callback) { } else { const value = child.value const label = labelPrefix + child.key - links.push(new DAGLink(label, value.size, value.multihash)) + links.push(new DAGLink(label, value.size, value.cid)) callback() } } @@ -176,7 +180,7 @@ function flush (options, bucket, path, ipld, source, callback) { const pushable = { path: path, size: node.size, - multihash: cid.buffer + cid: cid } if (source) { source.push(pushable) diff --git a/src/importer/index.js b/src/importer/index.js index 7b44c55..1bf0f0f 100644 --- a/src/importer/index.js +++ b/src/importer/index.js @@ -108,6 +108,7 @@ module.exports = function (ipld, _options) { callback(err) return } + pausable.resume() callback(null, hash) }) diff --git a/src/importer/tree-builder.js b/src/importer/tree-builder.js index 386fb5c..fc47844 100644 --- a/src/importer/tree-builder.js +++ b/src/importer/tree-builder.js @@ -10,6 +10,7 @@ const DirFlat = require('./dir-flat') const flatToShard = require('./flat-to-shard') const Dir = require('./dir') const toPathComponents = require('../utils/to-path-components') +const errCode = require('err-code') module.exports = createTreeBuilder @@ -104,7 +105,7 @@ function createTreeBuilder (ipld, _options) { currentPath += pathElem const last = (index === lastIndex) parent.dirty = true - parent.multihash = null + parent.cid = null parent.size = null if (last) { @@ -151,7 +152,7 @@ function createTreeBuilder (ipld, _options) { if (err) { callback(err) } else { - callback(null, node && node.multihash) + callback(null, node && node.cid) } } }) @@ -160,7 +161,7 @@ function createTreeBuilder (ipld, _options) { function flush (path, tree, callback) { if (tree.dir) { if (tree.root && tree.childCount() > 1 && !options.wrap) { - callback(new Error('detected more than one root')) + callback(errCode(new Error('detected more than one root'), 'EMORETHANONEROOT')) return // early } tree.eachChildSeries( @@ -196,7 +197,7 @@ function createTreeBuilder (ipld, _options) { } if (!tree.dirty) { - callback(null, tree.multihash) + callback(null, tree.cid) return // early } diff --git a/src/index.js b/src/index.js index 2e8122b..5963cf3 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,26 @@ 'use strict' -module.exports = require('./importer') +const pull = require('pull-stream/pull') +const map = require('pull-stream/throughs/map') +const toPull = require('async-iterator-to-pull-stream') +const toIterator = require('pull-stream-to-async-iterator') +const importer = require('./importer') + +module.exports = function (source, ipld, options = {}) { + return toIterator( + pull( + toPull.source(source), + map(({ path, content }) => { + if (content && content[Symbol.asyncIterator]) { + content = toPull(content) + } + + return { + path, + content + } + }), + importer(ipld, options) + ) + ) +} diff --git a/src/utils/persist.js b/src/utils/persist.js index 5fa987c..039361b 100644 --- a/src/utils/persist.js +++ b/src/utils/persist.js @@ -1,5 +1,8 @@ 'use strict' +const mh = require('multihashes') +const mc = require('multicodec') + const { util: { cid @@ -22,10 +25,14 @@ const persist = (node, ipld, options, callback) => { codec = 'raw' } - if (hashAlg !== 'sha2-256') { + if (hashAlg !== 'sha2-256' && hashAlg !== mh.names['sha2-256']) { cidVersion = 1 } + if (isNaN(hashAlg)) { + hashAlg = mh.names[hashAlg] + } + if (options.onlyHash) { return cid(node, { version: cidVersion, @@ -38,16 +45,14 @@ const persist = (node, ipld, options, callback) => { }) } - ipld.put(node, { - version: cidVersion, - hashAlg: hashAlg, - format: codec - }, (error, cid) => { - callback(error, { + ipld.put(node, mc[codec.toUpperCase().replace(/-/g, '_')], { + cidVersion: cidVersion, + hashAlg: hashAlg + }) + .then((cid) => callback(null, { cid, node - }) - }) + }), callback) } module.exports = persist diff --git a/test/benchmark.spec.js b/test/benchmark.spec.js index 71eb98b..42314e3 100644 --- a/test/benchmark.spec.js +++ b/test/benchmark.spec.js @@ -6,12 +6,9 @@ const importer = require('../src') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const onEnd = require('pull-stream/sinks/on-end') const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const bufferStream = require('pull-buffer-stream') +const bufferStream = require('async-iterator-buffer-stream') const REPEATS = 10 const FILE_SIZE = Math.pow(2, 20) * 500 // 500MB @@ -35,14 +32,14 @@ describe.skip('benchmark', function () { const times = [] after(() => { - console.info(`Percent\tms`) + console.info(`Percent\tms`) // eslint-disable-line no-console times.forEach((time, index) => { - console.info(`${index}\t${parseInt(time / REPEATS)}`) + console.info(`${index}\t${parseInt(time / REPEATS)}`) // eslint-disable-line no-console }) }) for (let i = 0; i < REPEATS; i++) { - it(`run ${i}`, (done) => { // eslint-disable-line no-loop-func + it(`run ${i}`, async () => { // eslint-disable-line no-loop-func this.timeout(0) const size = FILE_SIZE @@ -67,22 +64,17 @@ describe.skip('benchmark', function () { const buf = Buffer.alloc(CHUNK_SIZE).fill(0) - pull( - values([{ - path: '200Bytes.txt', - content: bufferStream(size, { - chunkSize: CHUNK_SIZE, - generator: (num, cb) => { - cb(null, buf) - } - }) - }]), - importer(ipld, options), - onEnd((err) => { - expect(err).to.not.exist() - done() + for await (const file of importer({ // eslint-disable-line no-unused-vars + path: '200Bytes.txt', + content: bufferStream(size, { + chunkSize: CHUNK_SIZE, + generator: () => { + return buf + } }) - ) + }, ipld, options)) { + // do nothing + } }) } }) diff --git a/test/builder-balanced.spec.js b/test/builder-balanced.spec.js index 73fe7e6..24c19c4 100644 --- a/test/builder-balanced.spec.js +++ b/test/builder-balanced.spec.js @@ -6,8 +6,18 @@ chai.use(require('dirty-chai')) const expect = chai.expect const pull = require('pull-stream/pull') const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') -const builder = require('../src/builder/balanced') +const toIterator = require('pull-stream-to-async-iterator') +const pullBuilder = require('../src/builder/balanced') +const all = require('async-iterator-all') + +const builder = (source, reduce, options) => { + return toIterator( + pull( + values(source), + pullBuilder(reduce, options) + ) + ) +} function reduce (leaves, callback) { if (leaves.length > 1) { @@ -22,74 +32,51 @@ const options = { } describe('builder: balanced', () => { - it('reduces one value into itself', (callback) => { - pull( - values([1]), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([1]) - callback() - }) - ) + it('reduces one value into itself', async () => { + const source = [1] + + const result = await all(builder(source, reduce, options)) + + expect(result).to.deep.equal(source) }) - it('reduces 3 values into parent', (callback) => { - pull( - values([1, 2, 3]), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([{ - children: [1, 2, 3] - }]) - callback() - }) - ) + it('reduces 3 values into parent', async () => { + const source = [1, 2, 3] + + const result = await all(builder(source, reduce, options)) + + expect(result).to.deep.equal([{ + children: [1, 2, 3] + }]) }) - it('obeys max children per node', (callback) => { - pull( - values([1, 2, 3, 4]), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([ - { - children: [ - { - children: [1, 2, 3] - }, - 4 - ] - } - ]) - callback() - }) - ) + it('obeys max children per node', async () => { + const source = [1, 2, 3, 4] + + const result = await all(builder(source, reduce, options)) + + expect(result).to.deep.equal([{ + children: [{ + children: [1, 2, 3] + }, + 4 + ] + }]) }) - it('refolds 2 parent nodes', (callback) => { - pull( - values([1, 2, 3, 4, 5, 6, 7]), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([ - { - children: [ - { - children: [1, 2, 3] - }, - { - children: [4, 5, 6] - }, - 7 - ] - } - ]) - callback() - }) - ) + it('refolds 2 parent nodes', async () => { + const source = [1, 2, 3, 4, 5, 6, 7] + + const result = await all(builder(source, reduce, options)) + + expect(result).to.deep.equal([{ + children: [{ + children: [1, 2, 3] + }, { + children: [4, 5, 6] + }, + 7 + ] + }]) }) }) diff --git a/test/builder-dir-sharding.spec.js b/test/builder-dir-sharding.spec.js index 343fca6..bd604f7 100644 --- a/test/builder-dir-sharding.spec.js +++ b/test/builder-dir-sharding.spec.js @@ -9,15 +9,8 @@ chai.use(require('dirty-chai')) const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const asyncMap = require('pull-stream/throughs/async-map') -const collect = require('pull-stream/sinks/collect') -const pushable = require('pull-pushable') -const whilst = require('async/whilst') -const setImmediate = require('async/setImmediate') const leftPad = require('left-pad') -const CID = require('cids') +const all = require('async-iterator-all') describe('builder: directory sharding', () => { let ipld @@ -33,135 +26,94 @@ describe('builder: directory sharding', () => { }) describe('basic dirbuilder', () => { - let nonShardedHash, shardedHash - - it('yields a non-sharded dir', (done) => { - const options = { + it('yields a non-sharded dir', async () => { + const nodes = await all(importer([{ + path: 'a/b', + content: Buffer.from('i have the best bytes') + }], ipld, { shardSplitThreshold: Infinity // never shard - } + })) - pull( - values([ - { - path: 'a/b', - content: values([Buffer.from('i have the best bytes')]) - } - ]), - importer(ipld, options), - collect((err, nodes) => { - try { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(2) - expect(nodes[0].path).to.be.eql('a/b') - expect(nodes[1].path).to.be.eql('a') - nonShardedHash = nodes[1].multihash - expect(nonShardedHash).to.exist() - done() - } catch (err) { - done(err) - } - }) - ) + expect(nodes.length).to.equal(2) + expect(nodes[0].path).to.equal('a/b') + expect(nodes[1].path).to.equal('a') + + const node = await exporter(nodes[1].cid, ipld) + + expect(node.unixfs.type).to.equal('directory') }) - it('yields a sharded dir', (done) => { - const options = { + it('yields a sharded dir', async () => { + const nodes = await all(importer([{ + path: 'a/b', + content: Buffer.from('i have the best bytes') + }], ipld, { shardSplitThreshold: 0 // always shard - } + })) - pull( - values([ - { - path: 'a/b', - content: values([Buffer.from('i have the best bytes')]) - } - ]), - importer(ipld, options), - collect((err, nodes) => { - try { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(2) - expect(nodes[0].path).to.be.eql('a/b') - expect(nodes[1].path).to.be.eql('a') - shardedHash = nodes[1].multihash - // hashes are different - expect(shardedHash).to.not.equal(nonShardedHash) - done() - } catch (err) { - done(err) - } - }) - ) + expect(nodes.length).to.equal(2) + expect(nodes[0].path).to.equal('a/b') + expect(nodes[1].path).to.equal('a') + + const node = await exporter(nodes[1].cid, ipld) + + expect(node.unixfs.type).to.equal('hamt-sharded-directory') }) - it('exporting unsharded hash results in the correct files', (done) => { - pull( - exporter(nonShardedHash, ipld), - collect((err, nodes) => { - try { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(2) - const expectedHash = new CID(nonShardedHash).toBaseEncodedString() - expect(nodes[0].path).to.be.eql(expectedHash) - expect(nodes[0].cid.toBaseEncodedString()).to.be.eql(expectedHash) - expect(nodes[1].path).to.be.eql(expectedHash + '/b') - expect(nodes[1].size).to.be.eql(21) - } catch (err) { - return done(err) - } + it('exporting unsharded hash results in the correct files', async () => { + const content = 'i have the best bytes' + const nodes = await all(importer([{ + path: 'a/b', + content: Buffer.from(content) + }], ipld, { + shardSplitThreshold: Infinity // never shard + })) - pull( - nodes[1].content, - collect(collected) - ) - }) - ) - - function collected (err, content) { - try { - expect(err).to.not.exist() - expect(content.length).to.be.eql(1) - expect(content[0].toString()).to.be.eql('i have the best bytes') - done() - } catch (err) { - done(err) - } - } + const nonShardedHash = nodes[1].cid + + const dir = await exporter(nonShardedHash, ipld) + const files = await all(dir.content()) + + expect(files.length).to.equal(1) + + const expectedHash = nonShardedHash.toBaseEncodedString() + + expect(dir.path).to.be.eql(expectedHash) + expect(dir.cid.toBaseEncodedString()).to.be.eql(expectedHash) + expect(files[0].path).to.be.eql(expectedHash + '/b') + expect(files[0].unixfs.fileSize()).to.be.eql(content.length) + + const fileContent = Buffer.concat(await all(files[0].content())) + + expect(fileContent.toString()).to.equal(content) }) - it('exporting sharded hash results in the correct files', (done) => { - pull( - exporter(shardedHash, ipld), - collect((err, nodes) => { - try { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(2) - const expectedHash = new CID(shardedHash).toBaseEncodedString() - expect(nodes[0].path).to.be.eql(expectedHash) - expect(nodes[0].cid.toBaseEncodedString()).to.be.eql(expectedHash) - expect(nodes[1].path).to.be.eql(expectedHash + '/b') - expect(nodes[1].size).to.be.eql(21) - } catch (err) { - return done(err) - } + it('exporting sharded hash results in the correct files', async () => { + const content = 'i have the best bytes' + const nodes = await all(importer([{ + path: 'a/b', + content: Buffer.from(content) + }], ipld, { + shardSplitThreshold: 0 // always shard + })) - pull( - nodes[1].content, - collect(collected) - ) - }) - ) - - function collected (err, content) { - try { - expect(err).to.not.exist() - expect(content.length).to.be.eql(1) - expect(content[0].toString()).to.be.eql('i have the best bytes') - done() - } catch (err) { - done(err) - } - } + const shardedHash = nodes[1].cid + + const dir = await exporter(shardedHash, ipld) + const files = await all(dir.content()) + + expect(files.length).to.equal(1) + + const expectedHash = shardedHash.toBaseEncodedString() + + expect(dir.path).to.be.eql(expectedHash) + expect(dir.cid.toBaseEncodedString()).to.be.eql(expectedHash) + expect(files[0].path).to.be.eql(expectedHash + '/b') + expect(files[0].unixfs.fileSize()).to.be.eql(content.length) + + const fileContent = Buffer.concat(await all(files[0].content())) + + expect(fileContent.toString()).to.equal(content) }) }) @@ -169,96 +121,48 @@ describe('builder: directory sharding', () => { this.timeout(30 * 1000) const maxDirs = 2000 - let rootHash - it('imports a big dir', (done) => { - const push = pushable() - pull( - push, - importer(ipld), - collect((err, nodes) => { - try { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(maxDirs + 1) - const last = nodes[nodes.length - 1] - expect(last.path).to.be.eql('big') - rootHash = last.multihash - done() - } catch (err) { - done(err) - } - }) - ) - - let pending = maxDirs - let i = 0 - - whilst( - () => pending, - (callback) => { - pending-- - i++ - const pushable = { - path: 'big/' + leftPad(i.toString(), 4, '0'), - content: values([Buffer.from(i.toString())]) + it('imports a big dir', async () => { + const source = { + [Symbol.asyncIterator]: async function * () { + for (let i = 0; i < maxDirs; i++) { + yield { + path: 'big/' + leftPad(i.toString(), 4, '0'), + content: Buffer.from(i.toString()) + } } - push.push(pushable) - setImmediate(callback) - }, - (err) => { - expect(err).to.not.exist() - push.end() } - ) - }) + } - it('exports a big dir', (done) => { - const contentEntries = [] - const entries = {} - pull( - exporter(rootHash, ipld), - asyncMap((node, callback) => { - if (node.content) { - pull( - node.content, - collect(collected) - ) - } else { - entries[node.path] = node - callback() - } + const nodes = await all(importer(source, ipld)) - function collected (err, content) { - expect(err).to.not.exist() - entries[node.path] = { content: content.toString() } - callback(null, node) + expect(nodes.length).to.equal(maxDirs + 1) + const last = nodes[nodes.length - 1] + expect(last.path).to.equal('big') + }) + + it('exports a big dir', async () => { + const source = { + [Symbol.asyncIterator]: async function * () { + for (let i = 0; i < maxDirs; i++) { + yield { + path: 'big/' + leftPad(i.toString(), 4, '0'), + content: Buffer.from(i.toString()) + } } - }), - collect((err, nodes) => { - expect(err).to.not.exist() - const paths = Object.keys(entries).sort() - expect(paths.length).to.be.eql(2001) - paths.forEach(eachPath) - done() - }) - ) - - function eachPath (path, index) { - if (!index) { - // first dir - expect(path).to.be.eql(new CID(rootHash).toBaseEncodedString()) - const entry = entries[path] - expect(entry).to.exist() - expect(entry.content).to.not.exist() - return - } - // dir entries - const content = entries[path] && entries[path].content - if (content) { - expect(content).to.be.eql(index.toString()) - contentEntries.push(path) } } + + const nodes = await all(importer(source, ipld)) + + expect(nodes.length).to.equal(maxDirs + 1) // files plus the containing directory + + const dir = await exporter(nodes[nodes.length - 1].cid, ipld) + + for await (const entry of dir.content()) { + const content = Buffer.concat(await all(entry.content())) + expect(content.toString()).to.equal(parseInt(entry.name, 10).toString()) + } }) }) @@ -269,135 +173,123 @@ describe('builder: directory sharding', () => { const maxDepth = 3 let rootHash - it('imports a big dir', (done) => { - const push = pushable() - pull( - push, - importer(ipld), - collect((err, nodes) => { - expect(err).to.not.exist() - const last = nodes[nodes.length - 1] - expect(last.path).to.be.eql('big') - rootHash = last.multihash - done() - }) - ) - - let pending = maxDirs - let pendingDepth = maxDepth - let i = 0 - let depth = 1 + before(async () => { + const source = { + [Symbol.asyncIterator]: async function * () { + let pending = maxDirs + let pendingDepth = maxDepth + let i = 0 + let depth = 1 + + while (pendingDepth && pending) { + i++ + const dir = [] + for (let d = 0; d < depth; d++) { + dir.push('big') + } - whilst( - () => pendingDepth && pending, - (callback) => { - i++ - const dir = [] - for (let d = 0; d < depth; d++) { - dir.push('big') - } - const pushed = { - path: dir.concat(leftPad(i.toString(), 4, '0')).join('/'), - content: values([Buffer.from(i.toString())]) - } - push.push(pushed) - pending-- - if (!pending) { - pendingDepth-- - pending = maxDirs - i = 0 - depth++ + yield { + path: dir.concat(leftPad(i.toString(), 4, '0')).join('/'), + content: Buffer.from(i.toString()) + } + + pending-- + if (!pending) { + pendingDepth-- + pending = maxDirs + i = 0 + depth++ + } } - setImmediate(callback) - }, - (err) => { - expect(err).to.not.exist() - push.end() } - ) + } + + const nodes = await all(importer(source, ipld)) + const last = nodes[nodes.length - 1] + expect(last.path).to.equal('big') + + rootHash = last.cid }) - it('exports a big dir', (done) => { - const entries = {} - pull( - exporter(rootHash, ipld), - asyncMap((node, callback) => { - if (node.content) { - pull( - node.content, - collect(collected) - ) - } else { - entries[node.path] = node - callback() + it('imports a big dir', async () => { + const dir = await exporter(rootHash, ipld) + + const verifyContent = async (node) => { + if (node.unixfs.type === 'file') { + const bufs = await all(node.content()) + const content = Buffer.concat(bufs) + expect(content.toString()).to.equal(parseInt(node.name, 10).toString()) + } else { + for await (const entry of node.content()) { + await verifyContent(entry) } + } + } - function collected (err, content) { - expect(err).to.not.exist() - entries[node.path] = { content: content.toString() } - callback(null, node) + await verifyContent(dir) + }) + + it('exports a big dir', async () => { + const collectContent = async (node, entries = {}) => { + if (node.unixfs.type === 'file') { + entries[node.path] = { + content: Buffer.concat(await all(node.content())).toString() } - }), - collect(collected) - ) - - function collected (err, nodes) { - expect(err).to.not.exist() - const paths = Object.keys(entries).sort() - expect(paths.length).to.be.eql(maxDepth * maxDirs + maxDepth) - let index = 0 - let depth = 1 - paths.forEach(eachPath) - done() - - function eachPath (path) { - if (!index) { - // first dir - if (depth === 1) { - expect(path).to.be.eql(new CID(rootHash).toBaseEncodedString()) - } - const entry = entries[path] - expect(entry).to.exist() - expect(entry.content).to.not.exist() - } else { - // dir entries - const pathElements = path.split('/') - expect(pathElements.length).to.be.eql(depth + 1) - const lastElement = pathElements[pathElements.length - 1] - expect(lastElement).to.be.eql(leftPad(index.toString(), 4, '0')) - expect(entries[path].content).to.be.eql(index.toString()) + } else { + entries[node.path] = node + + for await (const entry of node.content()) { + await collectContent(entry, entries) } - index++ - if (index > maxDirs) { - index = 0 - depth++ + } + + return entries + } + + const eachPath = (path) => { + if (!index) { + // first dir + if (depth === 1) { + expect(path).to.equal(dir.cid.toBaseEncodedString()) } + + const entry = entries[path] + expect(entry).to.exist() + expect(entry.content).to.not.be.a('string') + } else { + // dir entries + const pathElements = path.split('/') + expect(pathElements.length).to.equal(depth + 1) + const lastElement = pathElements[pathElements.length - 1] + expect(lastElement).to.equal(leftPad(index.toString(), 4, '0')) + expect(entries[path].content).to.equal(index.toString()) + } + index++ + if (index > maxDirs) { + index = 0 + depth++ } } + + const dir = await exporter(rootHash, ipld) + + const entries = await collectContent(dir) + let index = 0 + let depth = 1 + + const paths = Object.keys(entries).sort() + expect(paths.length).to.equal(maxDepth * maxDirs + maxDepth) + paths.forEach(eachPath) }) - it('exports a big dir with subpath', (done) => { - const exportHash = new CID(rootHash).toBaseEncodedString() + '/big/big/2000' - pull( - exporter(exportHash, ipld), - collect(collected) - ) - - function collected (err, nodes) { - expect(err).to.not.exist() - expect(nodes.length).to.equal(1) - expect(nodes.map((node) => node.path)).to.deep.equal([ - '2000' - ]) - pull( - nodes[0].content, - collect((err, content) => { - expect(err).to.not.exist() - expect(content.toString()).to.equal('2000') - done() - }) - ) - } + it('exports a big dir with subpath', async () => { + const exportHash = rootHash.toBaseEncodedString() + '/big/big/2000' + + const node = await exporter(exportHash, ipld) + expect(node.path).to.equal(exportHash) + + const content = Buffer.concat(await all(node.content())) + expect(content.toString()).to.equal('2000') }) }) }) diff --git a/test/builder-flat.spec.js b/test/builder-flat.spec.js index 8598864..b535f1f 100644 --- a/test/builder-flat.spec.js +++ b/test/builder-flat.spec.js @@ -6,9 +6,18 @@ chai.use(require('dirty-chai')) const expect = chai.expect const pull = require('pull-stream/pull') const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') +const toIterator = require('pull-stream-to-async-iterator') +const pullBuilder = require('../src/builder/flat') +const all = require('async-iterator-all') -const builder = require('../src/builder/flat') +const builder = (source, reduce, options) => { + return toIterator( + pull( + values(source), + pullBuilder(reduce, options) + ) + ) +} function reduce (leaves, callback) { if (leaves.length > 1) { @@ -19,27 +28,17 @@ function reduce (leaves, callback) { } describe('builder: flat', () => { - it('reduces one value into itself', (callback) => { - pull( - values([1]), - builder(reduce), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([1]) - callback() - }) - ) + it('reduces one value into itself', async () => { + const source = [1] + const result = await all(builder(source, reduce)) + + expect(result).to.be.eql([1]) }) - it('reduces 2 values into parent', (callback) => { - pull( - values([1, 2]), - builder(reduce), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.eql([{ children: [1, 2] }]) - callback() - }) - ) + it('reduces 2 values into parent', async () => { + const source = [1, 2] + const result = await all(builder(source, reduce)) + + expect(result).to.be.eql([{ children: [1, 2] }]) }) }) diff --git a/test/builder-only-hash.spec.js b/test/builder-only-hash.spec.js index 42fc4c8..33d0c58 100644 --- a/test/builder-only-hash.spec.js +++ b/test/builder-only-hash.spec.js @@ -6,12 +6,21 @@ chai.use(require('dirty-chai')) const expect = chai.expect const pull = require('pull-stream/pull') const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const CID = require('cids') const createBuilder = require('../src/builder') const FixedSizeChunker = require('../src/chunker/fixed-size') +const toIterator = require('pull-stream-to-async-iterator') +const all = require('async-iterator-all') + +const builder = (source, ipld, options) => { + return toIterator( + pull( + values(source), + createBuilder(FixedSizeChunker, ipld, options) + ) + ) +} describe('builder: onlyHash', () => { let ipld @@ -26,33 +35,22 @@ describe('builder: onlyHash', () => { }) }) - it('will only chunk and hash if passed an "onlyHash" option', (done) => { - const onCollected = (err, nodes) => { - if (err) return done(err) + it('will only chunk and hash if passed an "onlyHash" option', async () => { + const nodes = await all(builder({ + path: '/foo.txt', + content: Buffer.from([0, 1, 2, 3, 4]) + }, ipld, { + onlyHash: true + })) - const node = nodes[0] - expect(node).to.exist() + expect(nodes.length).to.equal(2) - ipld.get(new CID(node.multihash), (err, res) => { - expect(err).to.exist() - done() - }) - } + try { + await ipld.get(nodes[0].cid) - const content = String(Math.random() + Date.now()) - const inputFile = { - path: content + '.txt', - content: Buffer.from(content) + throw new Error('Should have errored') + } catch (err) { + expect(err.code).to.equal('ERR_NOT_FOUND') } - - const options = { - onlyHash: true - } - - pull( - values([inputFile]), - createBuilder(FixedSizeChunker, ipld, options), - collect(onCollected) - ) }) }) diff --git a/test/builder-trickle-dag.spec.js b/test/builder-trickle-dag.spec.js index 04ac623..ea07f43 100644 --- a/test/builder-trickle-dag.spec.js +++ b/test/builder-trickle-dag.spec.js @@ -5,11 +5,29 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const pull = require('pull-stream/pull') -const count = require('pull-stream/sources/count') const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') +const trickleBuilder = require('../src/builder/trickle') +const toIterator = require('pull-stream-to-async-iterator') +const all = require('async-iterator-all') -const builder = require('../src/builder/trickle') +const builder = (source, options) => { + return toIterator( + pull( + values(source), + trickleBuilder(reduce, options) + ) + ) +} + +const createValues = (max) => { + const output = [] + + for (let i = 0; i < max; i++) { + output.push(i) + } + + return output +} function reduce (leaves, callback) { if (leaves.length > 1) { @@ -25,446 +43,390 @@ const options = { } describe('builder: trickle', () => { - it('reduces one value into itself', callback => { - pull( - values([1]), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([1]) - callback() - }) - ) + it('reduces one value into itself', async () => { + const result = await all(builder([1], options)) + + expect(result).to.be.eql([1]) }) - it('reduces 3 values into parent', callback => { - pull( - count(2), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([ - { - children: [ - 0, - 1, - 2 - ] - } - ]) - callback() - }) - ) + it('reduces 3 values into parent', async () => { + const result = await all(builder(createValues(3), options)) + + expect(result).to.be.eql([{ + children: [ + 0, + 1, + 2 + ] + }]) }) - it('reduces 6 values correclty', callback => { - pull( - count(5), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([ - { - children: [ - 0, - 1, - 2, - { - children: [ - 3, - 4, - 5 - ] - } - ] - } - ]) - callback() - }) - ) + it('reduces 6 values correclty', async () => { + const result = await all(builder(createValues(6), options)) + + expect(result).to.be.eql([{ + children: [ + 0, + 1, + 2, + { + children: [ + 3, + 4, + 5 + ] + } + ] + }]) }) - it('reduces 9 values correclty', callback => { - pull( - count(8), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([ - { - children: [ - 0, - 1, - 2, - { - children: [ - 3, - 4, - 5 - ] - }, - { - children: [ - 6, - 7, - 8 - ] - } - ] - } - ]) - callback() - }) - ) + it('reduces 9 values correclty', async () => { + const result = await all(builder(createValues(9), options)) + + expect(result).to.be.eql([{ + children: [ + 0, + 1, + 2, + { + children: [ + 3, + 4, + 5 + ] + }, + { + children: [ + 6, + 7, + 8 + ] + } + ] + }]) }) - it('reduces 12 values correclty', callback => { - pull( - count(11), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - setTimeout(() => { - expect(result).to.be.eql([ + it('reduces 12 values correclty', async () => { + const result = await all(builder(createValues(12), options)) + + expect(result).to.be.eql([{ + children: [ + 0, + 1, + 2, + { + children: [ + 3, + 4, + 5 + ] + }, + { + children: [ + 6, + 7, + 8 + ] + }, + { + children: [ + 9, + 10, + 11 + ] + } + ] + }]) + }) + + it('reduces 21 values correclty', async () => { + const result = await all(builder(createValues(21), options)) + + expect(result).to.be.eql([{ + children: [ + 0, + 1, + 2, + { + children: [ + 3, + 4, + 5 + ] + }, + { + children: [ + 6, + 7, + 8 + ] + }, + { + children: [ + 9, + 10, + 11, { children: [ - 0, - 1, - 2, + 12, + 13, + 14 + ] + }, + { + children: [ + 15, + 16, + 17 + ] + } + ] + }, + { + children: [ + 18, + 19, + 20 + ] + } + ] + }]) + }) + + it('forms correct trickle tree', async () => { + const result = await all(builder(createValues(100), options)) + + expect(result).to.be.eql([{ + children: [ + 0, + 1, + 2, + { + children: [ + 3, + 4, + 5 + ] + }, + { + children: [ + 6, + 7, + 8 + ] + }, + { + children: [ + 9, + 10, + 11, + { + children: [ + 12, + 13, + 14 + ] + }, + { + children: [ + 15, + 16, + 17 + ] + } + ] + }, + { + children: [ + 18, + 19, + 20, + { + children: [ + 21, + 22, + 23 + ] + }, + { + children: [ + 24, + 25, + 26 + ] + } + ] + }, + { + children: [ + 27, + 28, + 29, + { + children: [ + 30, + 31, + 32 + ] + }, + { + children: [ + 33, + 34, + 35 + ] + }, + { + children: [ + 36, + 37, + 38, { children: [ - 3, - 4, - 5 + 39, + 40, + 41 ] }, { children: [ - 6, - 7, - 8 + 42, + 43, + 44 + ] + } + ] + }, + { + children: [ + 45, + 46, + 47, + { + children: [ + 48, + 49, + 50 ] }, { children: [ - 9, - 10, - 11 + 51, + 52, + 53 ] } ] } - ]) - callback() - }, 100) - }) - ) - }) - - it('reduces 21 values correclty', callback => { - pull( - count(20), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([ - { - children: [ - 0, - 1, - 2, - { - children: [ - 3, - 4, - 5 - ] - }, - { - children: [ - 6, - 7, - 8 - ] - }, - { - children: [ - 9, - 10, - 11, - { - children: [ - 12, - 13, - 14 - ] - }, - { - children: [ - 15, - 16, - 17 - ] - } - ] - }, - { - children: [ - 18, - 19, - 20 - ] - } - ] - } - ]) - callback() - }) - ) - }) - - it('forms correct trickle tree', callback => { - pull( - count(99), - builder(reduce, options), - collect((err, result) => { - expect(err).to.not.exist() - expect(result).to.be.eql([ - { - children: [ - 0, - 1, - 2, - { - children: [ - 3, - 4, - 5 - ] - }, - { - children: [ - 6, - 7, - 8 - ] - }, - { - children: [ - 9, - 10, - 11, - { - children: [ - 12, - 13, - 14 - ] - }, - { - children: [ - 15, - 16, - 17 - ] - } - ] - }, - { - children: [ - 18, - 19, - 20, - { - children: [ - 21, - 22, - 23 - ] - }, - { - children: [ - 24, - 25, - 26 - ] - } - ] - }, - { - children: [ - 27, - 28, - 29, - { - children: [ - 30, - 31, - 32 - ] - }, - { - children: [ - 33, - 34, - 35 - ] - }, - { - children: [ - 36, - 37, - 38, - { - children: [ - 39, - 40, - 41 - ] - }, - { - children: [ - 42, - 43, - 44 - ] - } - ] - }, - { - children: [ - 45, - 46, - 47, - { - children: [ - 48, - 49, - 50 - ] - }, - { - children: [ - 51, - 52, - 53 - ] - } - ] - } - ] - }, - { - children: [ - 54, - 55, - 56, - { - children: [ - 57, - 58, - 59 - ] - }, - { - children: [ - 60, - 61, - 62 - ] - }, - { - children: [ - 63, - 64, - 65, - { - children: [ - 66, - 67, - 68 - ] - }, - { - children: [ - 69, - 70, - 71 - ] - } - ] - }, - { - children: [ - 72, - 73, - 74, - { - children: [ - 75, - 76, - 77 - ] - }, - { - children: [ - 78, - 79, - 80 - ] - } - ] - } - ] - }, - { - children: [ - 81, - 82, - 83, - { - children: [ - 84, - 85, - 86 - ] - }, - { - children: [ - 87, - 88, - 89 - ] - }, - { - children: [ - 90, - 91, - 92, - { - children: [ - 93, - 94, - 95 - ] - }, - { - children: [ - 96, - 97, - 98 - ] - } - ] - }, - 99 - ] - } - ] - } - ]) - callback() - }) - ) + ] + }, + { + children: [ + 54, + 55, + 56, + { + children: [ + 57, + 58, + 59 + ] + }, + { + children: [ + 60, + 61, + 62 + ] + }, + { + children: [ + 63, + 64, + 65, + { + children: [ + 66, + 67, + 68 + ] + }, + { + children: [ + 69, + 70, + 71 + ] + } + ] + }, + { + children: [ + 72, + 73, + 74, + { + children: [ + 75, + 76, + 77 + ] + }, + { + children: [ + 78, + 79, + 80 + ] + } + ] + } + ] + }, + { + children: [ + 81, + 82, + 83, + { + children: [ + 84, + 85, + 86 + ] + }, + { + children: [ + 87, + 88, + 89 + ] + }, + { + children: [ + 90, + 91, + 92, + { + children: [ + 93, + 94, + 95 + ] + }, + { + children: [ + 96, + 97, + 98 + ] + } + ] + }, + 99 + ] + } + ] + }]) }) }) diff --git a/test/builder.spec.js b/test/builder.spec.js index 4a3c68b..1795664 100644 --- a/test/builder.spec.js +++ b/test/builder.spec.js @@ -6,15 +6,23 @@ chai.use(require('dirty-chai')) const expect = chai.expect const pull = require('pull-stream/pull') const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') const mh = require('multihashes') const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const eachSeries = require('async').eachSeries -const CID = require('cids') const UnixFS = require('ipfs-unixfs') const createBuilder = require('../src/builder') const FixedSizeChunker = require('../src/chunker/fixed-size') +const toIterator = require('pull-stream-to-async-iterator') +const first = require('async-iterator-first') + +const builder = (source, ipld, options) => { + return toIterator( + pull( + values(source), + createBuilder(FixedSizeChunker, ipld, options) + ) + ) +} describe('builder', () => { let ipld @@ -31,8 +39,9 @@ describe('builder', () => { const testMultihashes = Object.keys(mh.names).slice(1, 40) - it('allows multihash hash algorithm to be specified', (done) => { - eachSeries(testMultihashes, (hashAlg, cb) => { + it('allows multihash hash algorithm to be specified', async () => { + for (let i = 0; i < testMultihashes.length; i++) { + const hashAlg = testMultihashes[i] const options = { hashAlg, strategy: 'flat' } const content = String(Math.random() + Date.now()) const inputFile = { @@ -40,38 +49,26 @@ describe('builder', () => { content: Buffer.from(content) } - const onCollected = (err, nodes) => { - if (err) return cb(err) + const imported = await first(builder([Object.assign({}, inputFile)], ipld, options)) - const node = nodes[0] - expect(node).to.exist() + expect(imported).to.exist() - const cid = new CID(node.multihash) + // Verify multihash has been encoded using hashAlg + expect(mh.decode(imported.cid.multihash).name).to.equal(hashAlg) - // Verify multihash has been encoded using hashAlg - expect(mh.decode(cid.multihash).name).to.equal(hashAlg) + // Fetch using hashAlg encoded multihash + const node = await ipld.get(imported.cid) - // Fetch using hashAlg encoded multihash - ipld.get(cid, (err, res) => { - if (err) return cb(err) - const content = UnixFS.unmarshal(res.value.data).data - expect(content.equals(inputFile.content)).to.be.true() - cb() - }) - } - - pull( - values([Object.assign({}, inputFile)]), - createBuilder(FixedSizeChunker, ipld, options), - collect(onCollected) - ) - }, done) + const fetchedContent = UnixFS.unmarshal(node.data).data + expect(fetchedContent.equals(inputFile.content)).to.be.true() + } }) - it('allows multihash hash algorithm to be specified for big file', function (done) { + it('allows multihash hash algorithm to be specified for big file', async function () { this.timeout(30000) - eachSeries(testMultihashes, (hashAlg, cb) => { + for (let i = 0; i < testMultihashes.length; i++) { + const hashAlg = testMultihashes[i] const options = { hashAlg, strategy: 'flat' } const content = String(Math.random() + Date.now()) const inputFile = { @@ -80,63 +77,32 @@ describe('builder', () => { content: Buffer.alloc(262144 + 5).fill(1) } - const onCollected = (err, nodes) => { - if (err) return cb(err) - - const node = nodes[0] - - try { - expect(node).to.exist() - const cid = new CID(node.multihash) - expect(mh.decode(cid.multihash).name).to.equal(hashAlg) - } catch (err) { - return cb(err) - } - - cb() - } + const imported = await first(builder([Object.assign({}, inputFile)], ipld, options)) - pull( - values([Object.assign({}, inputFile)]), - createBuilder(FixedSizeChunker, ipld, options), - collect(onCollected) - ) - }, done) + expect(imported).to.exist() + expect(mh.decode(imported.cid.multihash).name).to.equal(hashAlg) + } }) - it('allows multihash hash algorithm to be specified for a directory', (done) => { - eachSeries(testMultihashes, (hashAlg, cb) => { + it('allows multihash hash algorithm to be specified for a directory', async () => { + for (let i = 0; i < testMultihashes.length; i++) { + const hashAlg = testMultihashes[i] + const options = { hashAlg, strategy: 'flat' } const inputFile = { path: `${String(Math.random() + Date.now())}-dir`, content: null } - const onCollected = (err, nodes) => { - if (err) return cb(err) - - const node = nodes[0] - - expect(node).to.exist() + const imported = await first(builder([Object.assign({}, inputFile)], ipld, options)) - const cid = new CID(node.multihash) + expect(mh.decode(imported.cid.multihash).name).to.equal(hashAlg) - expect(mh.decode(cid.multihash).name).to.equal(hashAlg) - - // Fetch using hashAlg encoded multihash - ipld.get(cid, (err, res) => { - if (err) return cb(err) - const meta = UnixFS.unmarshal(res.value.data) - expect(meta.type).to.equal('directory') - cb() - }) - } + // Fetch using hashAlg encoded multihash + const node = await ipld.get(imported.cid) - pull( - values([Object.assign({}, inputFile)]), - createBuilder(FixedSizeChunker, ipld, options), - collect(onCollected) - ) - }, done) + const meta = UnixFS.unmarshal(node.data) + expect(meta.type).to.equal('directory') + } }) }) diff --git a/test/chunker-fixed-size.spec.js b/test/chunker-fixed-size.spec.js index c973e91..ed3f0aa 100644 --- a/test/chunker-fixed-size.spec.js +++ b/test/chunker-fixed-size.spec.js @@ -1,19 +1,27 @@ /* eslint-env mocha */ 'use strict' -const chunker = require('../src/chunker/fixed-size') +const createChunker = require('../src/chunker/fixed-size') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const pull = require('pull-stream/pull') -const infinite = require('pull-stream/sources/infinite') const values = require('pull-stream/sources/values') -const take = require('pull-stream/throughs/take') -const collect = require('pull-stream/sinks/collect') -const loadFixture = require('aegir/fixtures') const isNode = require('detect-node') +const toIterator = require('pull-stream-to-async-iterator') +const all = require('async-iterator-all') +const loadFixture = require('aegir/fixtures') const rawFile = loadFixture('test/fixtures/1MiB.txt') +const chunker = (source, chunkSize) => { + return toIterator( + pull( + values(source), + createChunker(chunkSize) + ) + ) +} + describe('chunker: fixed size', function () { this.timeout(30000) @@ -23,7 +31,7 @@ describe('chunker: fixed size', function () { } }) - it('chunks non flat buffers', (done) => { + it('chunks non flat buffers', async () => { const b1 = Buffer.alloc(2 * 256) const b2 = Buffer.alloc(1 * 256) const b3 = Buffer.alloc(5 * 256) @@ -32,78 +40,56 @@ describe('chunker: fixed size', function () { b2.fill('b') b3.fill('c') - pull( - values([b1, b2, b3]), - chunker(256), - collect((err, chunks) => { - expect(err).to.not.exist() - expect(chunks).to.have.length(8) - chunks.forEach((chunk) => { - expect(chunk).to.have.length(256) - }) - done() - }) - ) + const chunks = await all(chunker([b1, b2, b3], 256)) + + expect(chunks).to.have.length(8) + chunks.forEach((chunk) => { + expect(chunk).to.have.length(256) + }) }) - it('256 Bytes chunks', (done) => { - pull( - infinite(() => Buffer.from('a')), - take(256 * 12), - chunker(256), - collect((err, chunks) => { - expect(err).to.not.exist() - expect(chunks).to.have.length(12) - chunks.forEach((chunk) => { - expect(chunk).to.have.length(256) - }) - done() - }) - ) + it('256 Bytes chunks', async () => { + const input = [] + const buf = Buffer.from('a') + + for (let i = 0; i < (256 * 12); i++) { + input.push(buf) + } + const chunks = await all(chunker(input, 256)) + + expect(chunks).to.have.length(12) + chunks.forEach((chunk) => { + expect(chunk).to.have.length(256) + }) }) - it('256 KiB chunks', (done) => { + it('256 KiB chunks', async () => { const KiB256 = 262144 + const chunks = await all(chunker([rawFile], KiB256)) - pull( - values([rawFile]), - chunker(KiB256), - collect((err, chunks) => { - expect(err).to.not.exist() - - expect(chunks).to.have.length(4) - chunks.forEach((chunk) => { - expect(chunk).to.have.length(KiB256) - }) - done() - }) - ) + expect(chunks).to.have.length(4) + chunks.forEach((chunk) => { + expect(chunk).to.have.length(KiB256) + }) }) - it('256 KiB chunks of non scalar filesize', (done) => { + it('256 KiB chunks of non scalar filesize', async () => { const KiB256 = 262144 let file = Buffer.concat([rawFile, Buffer.from('hello')]) - pull( - values([file]), - chunker(KiB256), - collect((err, chunks) => { - expect(err).to.not.exist() - - expect(chunks).to.have.length(5) - let counter = 0 - - chunks.forEach((chunk) => { - if (chunk.length < KiB256) { - counter++ - } else { - expect(chunk).to.have.length(KiB256) - } - }) - - expect(counter).to.equal(1) - done() - }) - ) + const chunks = await all(chunker([file], KiB256)) + + expect(chunks).to.have.length(5) + let counter = 0 + + chunks.forEach((chunk) => { + if (chunk.length < KiB256) { + counter++ + } else { + expect(chunk).to.have.length(KiB256) + } + }) + + expect(counter).to.equal(1) }) }) diff --git a/test/chunker-rabin-browser.spec.js b/test/chunker-rabin-browser.spec.js index fe4ca62..85148be 100644 --- a/test/chunker-rabin-browser.spec.js +++ b/test/chunker-rabin-browser.spec.js @@ -1,14 +1,24 @@ /* eslint-env mocha */ 'use strict' -const chunker = require('../src/chunker/rabin') +const createChunker = require('../src/chunker/rabin') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const pull = require('pull-stream/pull') const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') const isNode = require('detect-node') +const toIterator = require('pull-stream-to-async-iterator') +const all = require('async-iterator-all') + +const chunker = (source, options) => { + return toIterator( + pull( + values(source), + createChunker(options) + ) + ) +} describe('chunker: rabin browser', () => { before(function () { @@ -17,7 +27,7 @@ describe('chunker: rabin browser', () => { } }) - it('returns an error', function (done) { + it('returns an error', async () => { const b1 = Buffer.alloc(2 * 256) const b2 = Buffer.alloc(1 * 256) const b3 = Buffer.alloc(5 * 256) @@ -26,15 +36,14 @@ describe('chunker: rabin browser', () => { b2.fill('b') b3.fill('c') - pull( - values([b1, b2, b3]), - chunker({ minChunkSize: 48, avgChunkSize: 96, maxChunkSize: 192 }), - collect((err) => { - expect(err).to.exist() - expect(err.message).to.include('Rabin chunker not available') - - done() - }) - ) + try { + await all(chunker([b1, b2, b3], { + minChunkSize: 48, + avgChunkSize: 96, + maxChunkSize: 192 + })) + } catch (err) { + expect(err.message).to.include('Rabin chunker not available') + } }) }) diff --git a/test/chunker-rabin.spec.js b/test/chunker-rabin.spec.js index 29c1dfa..d33c9f4 100644 --- a/test/chunker-rabin.spec.js +++ b/test/chunker-rabin.spec.js @@ -1,19 +1,29 @@ /* eslint-env mocha */ 'use strict' -const chunker = require('../src/chunker/rabin') +const createChunker = require('../src/chunker/rabin') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const pull = require('pull-stream/pull') const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') const loadFixture = require('aegir/fixtures') const os = require('os') const isNode = require('detect-node') +const toIterator = require('pull-stream-to-async-iterator') +const all = require('async-iterator-all') const rawFile = loadFixture('test/fixtures/1MiB.txt') +const chunker = (source, options) => { + return toIterator( + pull( + values(source), + createChunker(options) + ) + ) +} + describe('chunker: rabin', function () { this.timeout(30000) @@ -27,7 +37,7 @@ describe('chunker: rabin', function () { } }) - it('chunks non flat buffers', (done) => { + it('chunks non flat buffers', async () => { const b1 = Buffer.alloc(2 * 256) const b2 = Buffer.alloc(1 * 256) const b3 = Buffer.alloc(5 * 256) @@ -36,38 +46,33 @@ describe('chunker: rabin', function () { b2.fill('b') b3.fill('c') - pull( - values([b1, b2, b3]), - chunker({ minChunkSize: 48, avgChunkSize: 96, maxChunkSize: 192 }), - collect((err, chunks) => { - expect(err).to.not.exist() - chunks.forEach((chunk) => { - expect(chunk).to.have.length.gte(48) - expect(chunk).to.have.length.lte(192) - }) - done() - }) - ) + const chunks = await all(chunker([b1, b2, b3], { + minChunkSize: 48, + avgChunkSize: 96, + maxChunkSize: 192 + })) + + chunks.forEach((chunk) => { + expect(chunk).to.have.length.gte(48) + expect(chunk).to.have.length.lte(192) + }) }) - it('uses default min and max chunk size when only avgChunkSize is specified', (done) => { + it('uses default min and max chunk size when only avgChunkSize is specified', async () => { const b1 = Buffer.alloc(10 * 256) b1.fill('a') - pull( - values([b1]), - chunker({ avgChunkSize: 256 }), - collect((err, chunks) => { - expect(err).to.not.exist() - chunks.forEach((chunk) => { - expect(chunk).to.have.length.gte(256 / 3) - expect(chunk).to.have.length.lte(256 * (256 / 2)) - }) - done() - }) - ) + + const chunks = await all(chunker([b1], { + avgChunkSize: 256 + })) + + chunks.forEach((chunk) => { + expect(chunk).to.have.length.gte(256 / 3) + expect(chunk).to.have.length.lte(256 * (256 / 2)) + }) }) - it('256 KiB avg chunks of non scalar filesize', (done) => { + it('256 KiB avg chunks of non scalar filesize', async () => { const KiB256 = 262144 let file = Buffer.concat([rawFile, Buffer.from('hello')]) const opts = { @@ -75,19 +80,12 @@ describe('chunker: rabin', function () { avgChunkSize: KiB256, maxChunkSize: KiB256 + (KiB256 / 2) } - pull( - values([file]), - chunker(opts), - collect((err, chunks) => { - expect(err).to.not.exist() - - chunks.forEach((chunk) => { - expect(chunk).to.have.length.gte(opts.minChunkSize) - expect(chunk).to.have.length.lte(opts.maxChunkSize) - }) - - done() - }) - ) + + const chunks = await all(chunker([file], opts)) + + chunks.forEach((chunk) => { + expect(chunk).to.have.length.gte(opts.minChunkSize) + expect(chunk).to.have.length.lte(opts.maxChunkSize) + }) }) }) diff --git a/test/hash-parity-with-go-ipfs.spec.js b/test/hash-parity-with-go-ipfs.spec.js index 5a39204..48d5539 100644 --- a/test/hash-parity-with-go-ipfs.spec.js +++ b/test/hash-parity-with-go-ipfs.spec.js @@ -6,13 +6,10 @@ const importer = require('../src') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') -const CID = require('cids') const IPLD = require('ipld') const inMemory = require('ipld-in-memory') const randomByteStream = require('./helpers/finite-pseudorandom-byte-stream') +const first = require('async-iterator-first') const strategies = [ 'flat', @@ -44,25 +41,17 @@ strategies.forEach(strategy => { }) }) - it('yields the same tree as go-ipfs', function (done) { + it('yields the same tree as go-ipfs', async function () { this.timeout(10 * 1000) - pull( - values([ - { - path: 'big.dat', - content: randomByteStream(45900000, 7382) - } - ]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(files.length).to.be.equal(1) - const file = files[0] - expect(new CID(file.multihash).toBaseEncodedString()).to.be.equal(expectedHashes[strategy]) - done() - }) - ) + const source = [{ + path: 'big.dat', + content: randomByteStream(45900000, 7382) + }] + + const file = await first(importer(source, ipld, options)) + + expect(file.cid.toBaseEncodedString()).to.be.equal(expectedHashes[strategy]) }) }) }) diff --git a/test/helpers/collect-leaf-cids.js b/test/helpers/collect-leaf-cids.js index 47cec1c..087a7ed 100644 --- a/test/helpers/collect-leaf-cids.js +++ b/test/helpers/collect-leaf-cids.js @@ -1,37 +1,20 @@ 'use strict' -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const asyncMap = require('pull-stream/throughs/async-map') -const filter = require('pull-stream/throughs/filter') -const flatten = require('pull-stream/throughs/flatten') -const collect = require('pull-stream/sinks/collect') -const traverse = require('pull-traverse') -const CID = require('cids') +module.exports = function (cid, ipld) { + async function * traverse (cid) { + const node = await ipld.get(cid) -module.exports = (ipld, multihash, callback) => { - pull( - traverse.depthFirst(new CID(multihash), (cid) => { - return pull( - values([cid]), - asyncMap((cid, callback) => { - ipld.get(cid, (error, result) => { - callback(error, !error && result.value) - }) - }), - asyncMap((node, callback) => { - if (!node.links) { - return callback() - } + if (Buffer.isBuffer(node) || !node.links.length) { + yield { + node, + cid + } - return callback( - null, node.links.map(link => link.cid) - ) - }), - filter(Boolean), - flatten() - ) - }), - collect(callback) - ) + return + } + + node.links.forEach(link => traverse(link.cid)) + } + + return traverse(cid) } diff --git a/test/helpers/stream-to-array.js b/test/helpers/stream-to-array.js new file mode 100644 index 0000000..e69de29 diff --git a/test/import-export-nested-dir.spec.js b/test/import-export-nested-dir.spec.js index 4a195cb..6efb003 100644 --- a/test/import-export-nested-dir.spec.js +++ b/test/import-export-nested-dir.spec.js @@ -6,12 +6,8 @@ chai.use(require('dirty-chai')) const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const pull = require('pull-stream/pull') const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') -const map = require('async/map') -const CID = require('cids') - +const all = require('async-iterator-all') const importer = require('../src') const exporter = require('ipfs-unixfs-exporter') @@ -29,100 +25,79 @@ describe('import and export: directory', () => { }) }) - it('imports', function (done) { + it('imports', async function () { this.timeout(20 * 1000) - pull( - values([ - { path: 'a/b/c/d/e', content: values([Buffer.from('banana')]) }, - { path: 'a/b/c/d/f', content: values([Buffer.from('strawberry')]) }, - { path: 'a/b/g', content: values([Buffer.from('ice')]) }, - { path: 'a/b/h', content: values([Buffer.from('cream')]) } - ]), - importer(ipld), - collect((err, files) => { - expect(err).to.not.exist() - expect(files.map(normalizeNode).sort(byPath)).to.be.eql([ - { path: 'a/b/h', - multihash: 'QmWHMpCtdNjemT2F3SjyrmnBXQXwEohaZd4apcbFBhbFRC' }, - { path: 'a/b/g', - multihash: 'QmQGwYzzTPcbqTiy2Nbp88gqqBqCWY4QZGfen45LFZkD5n' }, - { path: 'a/b/c/d/f', - multihash: 'QmNVHs2dy7AjGUotsubWVncRsD3SpRXm8MgmCCQTVdVACz' }, - { path: 'a/b/c/d/e', - multihash: 'QmYPbDKwc7oneCcEc6BcRSN5GXthTGWUCd19bTCyP9u3vH' }, - { path: 'a/b/c/d', - multihash: 'QmQGDXr3ysARM38n7h79Tx7yD3YxuzcnZ1naG71WMojPoj' }, - { path: 'a/b/c', - multihash: 'QmYTVcjYpN3hQLtJstCPE8hhEacAYjWAuTmmAAXoonamuE' }, - { path: 'a/b', - multihash: 'QmWyWYxq1GD9fEyckf5LrJv8hMW35CwfWwzDBp8bTw3NQj' }, - { path: 'a', - multihash: rootHash } - ]) - done() - }) - ) + const source = [ + { path: 'a/b/c/d/e', content: values([Buffer.from('banana')]) }, + { path: 'a/b/c/d/f', content: values([Buffer.from('strawberry')]) }, + { path: 'a/b/g', content: values([Buffer.from('ice')]) }, + { path: 'a/b/h', content: values([Buffer.from('cream')]) } + ] + + const files = await all(importer(source, ipld)) + + expect(files.map(normalizeNode).sort(byPath)).to.be.eql([ + { path: 'a/b/h', + multihash: 'QmWHMpCtdNjemT2F3SjyrmnBXQXwEohaZd4apcbFBhbFRC' }, + { path: 'a/b/g', + multihash: 'QmQGwYzzTPcbqTiy2Nbp88gqqBqCWY4QZGfen45LFZkD5n' }, + { path: 'a/b/c/d/f', + multihash: 'QmNVHs2dy7AjGUotsubWVncRsD3SpRXm8MgmCCQTVdVACz' }, + { path: 'a/b/c/d/e', + multihash: 'QmYPbDKwc7oneCcEc6BcRSN5GXthTGWUCd19bTCyP9u3vH' }, + { path: 'a/b/c/d', + multihash: 'QmQGDXr3ysARM38n7h79Tx7yD3YxuzcnZ1naG71WMojPoj' }, + { path: 'a/b/c', + multihash: 'QmYTVcjYpN3hQLtJstCPE8hhEacAYjWAuTmmAAXoonamuE' }, + { path: 'a/b', + multihash: 'QmWyWYxq1GD9fEyckf5LrJv8hMW35CwfWwzDBp8bTw3NQj' }, + { path: 'a', + multihash: rootHash } + ]) }) - it('exports', function (done) { + it('exports', async function () { this.timeout(20 * 1000) - pull( - exporter(rootHash, ipld), - collect((err, files) => { - expect(err).to.not.exist() - map( - files, - (file, callback) => { - if (file.content) { - pull( - file.content, - collect(mapFile(file, callback)) - ) - } else { - callback(null, { path: file.path }) - } - }, - (err, files) => { - expect(err).to.not.exist() - expect(files.filter(fileHasContent).sort(byPath)).to.eql([ - { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/h', - content: 'cream' }, - { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/g', - content: 'ice' }, - { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/f', - content: 'strawberry' }, - { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/e', - content: 'banana' } - ]) - done() - }) - }) - ) - - function mapFile (file, callback) { - return (err, fileContent) => { - callback(err, fileContent && { - path: file.path, - content: fileContent.toString() - }) - } - } + const dir = await exporter(rootHash, ipld) + const files = await recursiveExport(dir, rootHash) + + expect(files.sort(byPath)).to.eql([ + { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/h', + content: 'cream' }, + { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/g', + content: 'ice' }, + { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/f', + content: 'strawberry' }, + { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/e', + content: 'banana' } + ]) }) }) +async function recursiveExport (node, path, entries = []) { + for await (const entry of node.content()) { + if (entry.unixfs.type === 'directory') { + await recursiveExport(entry, `${path}/${entry.name}`, entries) + } else { + entries.push({ + path: `${path}/${entry.name}`, + content: Buffer.concat(await all(entry.content())).toString() + }) + } + } + + return entries +} + function normalizeNode (node) { return { path: node.path, - multihash: new CID(node.multihash).toBaseEncodedString() + multihash: node.cid.toBaseEncodedString() } } -function fileHasContent (file) { - return Boolean(file.content) -} - function byPath (a, b) { if (a.path > b.path) return -1 if (a.path < b.path) return 1 diff --git a/test/import-export.spec.js b/test/import-export.spec.js index c0a4820..b5763b7 100644 --- a/test/import-export.spec.js +++ b/test/import-export.spec.js @@ -7,12 +7,6 @@ chai.use(require('dirty-chai')) const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const concat = require('pull-stream/sinks/concat') -const flatten = require('pull-stream/throughs/flatten') -const map = require('pull-stream/throughs/map') -const collect = require('pull-stream/sinks/collect') const loadFixture = require('aegir/fixtures') const bigFile = loadFixture('test/fixtures/1.2MiB.txt') @@ -25,18 +19,6 @@ const strategies = [ 'trickle' ] -function fileEql (f1, fileData, callback) { - pull( - f1.content, - concat((err, data) => { - expect(err).to.not.exist() - // TODO: eql is super slow at comparing large buffers - // expect(data).to.eql(fileData) - callback() - }) - ) -} - describe('import and export', function () { this.timeout(30 * 1000) @@ -56,24 +38,17 @@ describe('import and export', function () { }) }) - it('import and export', (done) => { + it('imports and exports', async () => { const path = strategy + '-big.dat' + const values = [{ path: path, content: bigFile }] + + for await (const file of importer(values, ipld, importerOptions)) { + expect(file.path).to.eql(path) - pull( - values([{ path: path, content: values([bigFile]) }]), - importer(ipld, importerOptions), - map((file) => { - expect(file.path).to.eql(path) + const result = await exporter(file.cid, ipld) - return exporter(file.multihash, ipld) - }), - flatten(), - collect((err, files) => { - expect(err).to.not.exist() - expect(files[0].size).to.eql(bigFile.length) - fileEql(files[0], bigFile, done) - }) - ) + expect(result.unixfs.fileSize()).to.eql(bigFile.length) + } }) }) }) diff --git a/test/importer-flush.spec.js b/test/importer-flush.spec.js index cafc1fb..19abeeb 100644 --- a/test/importer-flush.spec.js +++ b/test/importer-flush.spec.js @@ -14,7 +14,7 @@ const map = require('pull-stream/throughs/map') const collect = require('pull-stream/sinks/collect') const pushable = require('pull-pushable') -describe('importer: flush', () => { +describe.skip('importer: flush', () => { let ipld before((done) => { diff --git a/test/importer.spec.js b/test/importer.spec.js index 8a8fc1f..08f2899 100644 --- a/test/importer.spec.js +++ b/test/importer.spec.js @@ -9,43 +9,32 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const spy = require('sinon/lib/sinon/spy') -const pull = require('pull-stream/pull') -const empty = require('pull-stream/sources/empty') -const once = require('pull-stream/sources/once') -const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') -const onEnd = require('pull-stream/sinks/on-end') -const CID = require('cids') const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const loadFixture = require('aegir/fixtures') -const each = require('async/each') -const waterfall = require('async/waterfall') -const parallel = require('async/parallel') const UnixFs = require('ipfs-unixfs') const collectLeafCids = require('./helpers/collect-leaf-cids') +const loadFixture = require('aegir/fixtures') +const bigFile = loadFixture('test/fixtures/1.2MiB.txt') +const smallFile = loadFixture('test/fixtures/200Bytes.txt') function stringifyMh (files) { return files.map((file) => { - file.multihash = new CID(file.multihash).toBaseEncodedString() + file.cid = file.cid.toBaseEncodedString() return file }) } -const bigFile = loadFixture('test/fixtures/1.2MiB.txt') -const smallFile = loadFixture('test/fixtures/200Bytes.txt') - const baseFiles = { '200Bytes.txt': { path: '200Bytes.txt', - multihash: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', + cid: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', size: 211, name: '', leafSize: 200 }, '1.2MiB.txt': { path: '1.2MiB.txt', - multihash: 'QmbPN6CXXWpejfQgnRYnMQcVYkFHEntHWqLNQjbkatYCh1', + cid: 'QmbPN6CXXWpejfQgnRYnMQcVYkFHEntHWqLNQjbkatYCh1', size: 1328062, name: '', leafSize: 1258000 @@ -56,13 +45,13 @@ const strategyBaseFiles = { flat: baseFiles, balanced: extend({}, baseFiles, { '1.2MiB.txt': { - multihash: 'QmeEGqUisUD2T6zU96PrZnCkHfXCGuQeGWKu4UoSuaZL3d', + cid: 'QmeEGqUisUD2T6zU96PrZnCkHfXCGuQeGWKu4UoSuaZL3d', size: 1335420 } }), trickle: extend({}, baseFiles, { '1.2MiB.txt': { - multihash: 'QmaiSohNUt1rBf2Lqz6ou54NHVPTbXbBoPuq9td4ekcBx4', + cid: 'QmaiSohNUt1rBf2Lqz6ou54NHVPTbXbBoPuq9td4ekcBx4', size: 1334599 } }) @@ -78,102 +67,80 @@ const strategyOverrides = { balanced: { 'foo-big': { path: 'foo-big', - multihash: 'QmQ1S6eEamaf4t948etp8QiYQ9avrKCogiJnPRgNkVreLv', + cid: 'QmQ1S6eEamaf4t948etp8QiYQ9avrKCogiJnPRgNkVreLv', size: 1335478 }, pim: { - multihash: 'QmUpzaN4Jio2GB3HoPSRCMQD5EagdMWjSEGD4SGZXaCw7W', + cid: 'QmUpzaN4Jio2GB3HoPSRCMQD5EagdMWjSEGD4SGZXaCw7W', size: 1335744 }, 'pam/pum': { - multihash: 'QmUpzaN4Jio2GB3HoPSRCMQD5EagdMWjSEGD4SGZXaCw7W', + cid: 'QmUpzaN4Jio2GB3HoPSRCMQD5EagdMWjSEGD4SGZXaCw7W', size: 1335744 }, pam: { - multihash: 'QmVoVD4fEWFLJLjvRCg4bGrziFhgECiaezp79AUfhuLgno', + cid: 'QmVoVD4fEWFLJLjvRCg4bGrziFhgECiaezp79AUfhuLgno', size: 2671269 } }, trickle: { 'foo-big': { path: 'foo-big', - multihash: 'QmPh6KSS7ghTqzgWhaoCiLoHFPF7HGqUxx7q9vcM5HUN4U', + cid: 'QmPh6KSS7ghTqzgWhaoCiLoHFPF7HGqUxx7q9vcM5HUN4U', size: 1334657 }, pim: { - multihash: 'QmPAn3G2x2nrq4A1fu2XUpwWtpqG4D1YXFDrU615NHvJbr', + cid: 'QmPAn3G2x2nrq4A1fu2XUpwWtpqG4D1YXFDrU615NHvJbr', size: 1334923 }, 'pam/pum': { - multihash: 'QmPAn3G2x2nrq4A1fu2XUpwWtpqG4D1YXFDrU615NHvJbr', + cid: 'QmPAn3G2x2nrq4A1fu2XUpwWtpqG4D1YXFDrU615NHvJbr', size: 1334923 }, pam: { - multihash: 'QmZTJah1xpG9X33ZsPtDEi1tYSHGDqQMRHsGV5xKzAR2j4', + cid: 'QmZTJah1xpG9X33ZsPtDEi1tYSHGDqQMRHsGV5xKzAR2j4', size: 2669627 } } } -const checkLeafNodeTypes = (ipld, options, expected, done) => { - waterfall([ - (cb) => pull( - once({ - path: '/foo', - content: Buffer.alloc(262144 + 5).fill(1) - }), - importer(ipld, options), - collect(cb) - ), - (files, cb) => ipld.get(new CID(files[0].multihash), cb), - (result, cb) => { - const node = result.value - const meta = UnixFs.unmarshal(node.data) - - expect(meta.type).to.equal('file') - expect(node.links.length).to.equal(2) - - parallel( - node.links.map(link => { - return (done) => { - waterfall([ - (next) => ipld.get(link.cid, next), - (result, next) => { - const node = result.value - const meta = UnixFs.unmarshal(node.data) - - expect(meta.type).to.equal(expected) - - next() - } - ], done) - } - }), cb) - } - ], done) +const checkLeafNodeTypes = async (ipld, options, expected, done) => { + const files = [] + + for await (const file of importer([{ + path: '/foo', + content: Buffer.alloc(262144 + 5).fill(1) + }], ipld, options)) { + files.push(file) + } + + const node = await ipld.get(files[0].cid) + const meta = UnixFs.unmarshal(node.data) + + expect(meta.type).to.equal('file') + expect(node.links.length).to.equal(2) + + const linkedNodes = await Promise.all( + node.links.map(link => ipld.get(link.cid)) + ) + + linkedNodes.forEach(node => { + const meta = UnixFs.unmarshal(node.data) + expect(meta.type).to.equal(expected) + }) } -const checkNodeLinks = (ipld, options, expected, done) => { - waterfall([ - (cb) => pull( - once({ - path: '/foo', - content: Buffer.alloc(100).fill(1) - }), - importer(ipld, options), - collect(cb) - ), - (files, cb) => ipld.get(new CID(files[0].multihash), cb), - (result, cb) => { - const node = result.value - const meta = UnixFs.unmarshal(node.data) - - expect(meta.type).to.equal('file') - expect(node.links.length).to.equal(expected) - - cb() - } - ], done) +const checkNodeLinks = async (ipld, options, expected) => { + for await (const file of importer([{ + path: '/foo', + content: Buffer.alloc(100).fill(1) + }], ipld, options)) { + const node = await ipld.get(file.cid) + const meta = UnixFs.unmarshal(node.data) + + expect(meta.type).to.equal('file') + expect(node.links.length).to.equal(expected) + } } strategies.forEach((strategy) => { @@ -184,12 +151,12 @@ strategies.forEach((strategy) => { }), foo: { path: 'foo', - multihash: 'QmQrb6KKWGo8w7zKfx2JksptY6wN7B2ysSBdKZr4xMU36d', + cid: 'QmQrb6KKWGo8w7zKfx2JksptY6wN7B2ysSBdKZr4xMU36d', size: 320 }, 'foo/bar': { path: 'foo/bar', - multihash: 'Qmf5BQbTUyUAvd6Ewct83GYGnE1F6btiC3acLhR8MDxgkD', + cid: 'Qmf5BQbTUyUAvd6Ewct83GYGnE1F6btiC3acLhR8MDxgkD', size: 270 }, 'foo-big/1.2MiB.txt': extend({}, baseFiles['1.2MiB.txt'], { @@ -197,7 +164,7 @@ strategies.forEach((strategy) => { }), 'foo-big': { path: 'foo-big', - multihash: 'Qma6JU3FoXU9eAzgomtmYPjzFBwVc2rRbECQpmHFiA98CJ', + cid: 'Qma6JU3FoXU9eAzgomtmYPjzFBwVc2rRbECQpmHFiA98CJ', size: 1328120 }, 'pim/200Bytes.txt': extend({}, baseFiles['200Bytes.txt'], { @@ -208,24 +175,24 @@ strategies.forEach((strategy) => { }), pim: { path: 'pim', - multihash: 'QmNk8VPGb3fkAQgoxctXo4Wmnr4PayFTASy4MiVXTtXqiA', + cid: 'QmNk8VPGb3fkAQgoxctXo4Wmnr4PayFTASy4MiVXTtXqiA', size: 1328386 }, 'empty-dir': { path: 'empty-dir', - multihash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn', + cid: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn', size: 4 }, 'pam/pum': { - multihash: 'QmNk8VPGb3fkAQgoxctXo4Wmnr4PayFTASy4MiVXTtXqiA', + cid: 'QmNk8VPGb3fkAQgoxctXo4Wmnr4PayFTASy4MiVXTtXqiA', size: 1328386 }, pam: { - multihash: 'QmPAixYTaYnPe795fcWcuRpo6tfwHgRKNiBHpMzoomDVN6', + cid: 'QmPAixYTaYnPe795fcWcuRpo6tfwHgRKNiBHpMzoomDVN6', size: 2656553 }, '200Bytes.txt with raw leaves': extend({}, baseFiles['200Bytes.txt'], { - multihash: 'zb2rhXrz1gkCv8p4nUDZRohY6MzBE9C3HVTVDP72g6Du3SD9Q', + cid: 'zb2rhXrz1gkCv8p4nUDZRohY6MzBE9C3HVTVDP72g6Du3SD9Q', size: 200 }) }, strategyOverrides[strategy]) @@ -254,370 +221,325 @@ strategies.forEach((strategy) => { }) }) - it('fails on bad input', (done) => { - pull( - values([{ + it('fails on bad input', async () => { + try { + for await (const _ of importer([{ // eslint-disable-line no-unused-vars path: '200Bytes.txt', content: 'banana' - }]), - importer(ipld, options), - onEnd((err) => { - expect(err).to.exist() - done() - }) - ) + }], ipld, options)) { + // ...falala + } + + throw new Error('No error was thrown') + } catch (err) { + expect(err.code).to.equal('EINVALIDCONTENT') + } }) - it('survives bad progress option', (done) => { - pull( - values([{ - path: '200Bytes.txt', - content: Buffer.from([0, 1, 2]) - }]), - importer(ipld, { - ...options, - progress: null - }), - onEnd((err) => { - expect(err).to.not.exist() - done() - }) - ) + it('survives bad progress option', async () => { + let file + + for await (const f of importer([{ + path: '200Bytes.txt', + content: Buffer.from([0, 1, 2]) + }], ipld, { + ...options, + progress: null + })) { + file = f + } + + expect(file).to.be.ok() }) - it('doesn\'t yield anything on empty source', (done) => { - pull( - empty(), - importer(ipld, options), - collect((err, nodes) => { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(0) - done() - })) + it('doesn\'t yield anything on empty source', async () => { + const files = [] + + for await (const file of importer([], ipld, options)) { + files.push(file) + } + + expect(files).to.be.empty() }) - it('doesn\'t yield anything on empty file', (done) => { - pull( - values([{ - path: 'emptyfile', - content: empty() - }]), - importer(ipld, options), - collect((err, nodes) => { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(1) - - // always yield empty node - expect(new CID(nodes[0].multihash).toBaseEncodedString()).to.be.eql('QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH') - done() - })) + it('doesn\'t yield anything on empty file', async () => { + const files = [] + + for await (const file of importer([{ + path: 'emptyfile', + content: Buffer.alloc(0) + }], ipld, options)) { + files.push(file) + } + + expect(files.length).to.eql(1) + + // always yield empty node + expect(files[0].cid.toBaseEncodedString()).to.eql('QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH') }) - it('fails on more than one root', (done) => { - pull( - values([ - { - path: '/beep/200Bytes.txt', - content: values([smallFile]) - }, - { - path: '/boop/200Bytes.txt', - content: values([bigFile]) - } - ]), - importer(ipld, options), - onEnd((err) => { - expect(err).to.exist() - expect(err.message).to.be.eql('detected more than one root') - done() - }) - ) + it('fails on more than one root', async () => { + try { + for await (const _ of importer([{ // eslint-disable-line no-unused-vars + path: '/beep/200Bytes.txt', + content: smallFile + }, { + path: '/boop/200Bytes.txt', + content: bigFile + }], ipld, options)) { + // ...falala + } + + throw new Error('No error was thrown') + } catch (err) { + expect(err.code).to.equal('EMORETHANONEROOT') + } }) - it('small file with an escaped slash in the title', (done) => { + it('small file with an escaped slash in the title', async () => { const filePath = `small-\\/file-${Math.random()}.txt` + const files = [] - pull( - values([{ - path: filePath, - content: values([smallFile]) - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(files.length).to.equal(1) - expect(files[0].path).to.equal(filePath) - done() - }) - ) + for await (const file of importer([{ + path: filePath, + content: smallFile + }], ipld, options)) { + files.push(file) + } + + expect(files.length).to.equal(1) + expect(files[0].path).to.equal(filePath) }) - it('small file with square brackets in the title', (done) => { + it('small file with square brackets in the title', async () => { const filePath = `small-[v]-file-${Math.random()}.txt` + const files = [] - pull( - values([{ - path: filePath, - content: values([smallFile]) - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(files.length).to.equal(1) - expect(files[0].path).to.equal(filePath) - done() - }) - ) + for await (const file of importer([{ + path: filePath, + content: smallFile + }], ipld, options)) { + files.push(file) + } + + expect(files.length).to.equal(1) + expect(files[0].path).to.equal(filePath) }) - it('small file (smaller than a chunk)', (done) => { - pull( - values([{ - path: '200Bytes.txt', - content: values([smallFile]) - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt']]) - done() - }) - ) + it('small file (smaller than a chunk)', async () => { + const files = [] + + for await (const file of importer([{ + path: '200Bytes.txt', + content: smallFile + }], ipld, options)) { + files.push(file) + } + + expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt']]) }) - it('small file (smaller than a chunk) with raw leaves', (done) => { - pull( - values([{ - path: '200Bytes.txt', - content: values([smallFile]) - }]), - importer(ipld, Object.assign({}, options, { rawLeaves: true })), - collect((err, files) => { - expect(err).to.not.exist() - expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt with raw leaves']]) - done() - }) - ) + it('small file (smaller than a chunk) with raw leaves', async () => { + const files = [] + + for await (const file of importer([{ + path: '200Bytes.txt', + content: smallFile + }], ipld, { + ...options, + rawLeaves: true + })) { + files.push(file) + } + + expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt with raw leaves']]) }) - it('small file as buffer (smaller than a chunk)', (done) => { - pull( - values([{ - path: '200Bytes.txt', - content: smallFile - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt']]) - done() - }) - ) + it('small file as buffer (smaller than a chunk)', async () => { + const files = [] + + for await (const file of importer([{ + path: '200Bytes.txt', + content: smallFile + }], ipld, options)) { + files.push(file) + } + + expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt']]) }) - it('small file (smaller than a chunk) inside a dir', (done) => { - pull( - values([{ - path: 'foo/bar/200Bytes.txt', - content: values([smallFile]) - }]), - importer(ipld, options), - collect(collected) - ) + it('small file (smaller than a chunk) inside a dir', async () => { + const files = [] - function collected (err, files) { - expect(err).to.not.exist() - expect(files.length).to.equal(3) - stringifyMh(files).forEach((file) => { - if (file.path === 'foo/bar/200Bytes.txt') { - expect(file).to.be.eql(expected['foo/bar/200Bytes.txt']) - } - if (file.path === 'foo') { - expect(file).to.be.eql(expected.foo) - } - if (file.path === 'foo/bar') { - expect(file).to.be.eql(expected['foo/bar']) - } - }) - done() + for await (const file of importer([{ + path: 'foo/bar/200Bytes.txt', + content: smallFile + }], ipld, options)) { + files.push(file) } + + expect(files.length).to.equal(3) + stringifyMh(files).forEach((file) => { + expect(file).to.deep.equal(expected[file.path]) + }) }) - it('file bigger than a single chunk', function (done) { + it('file bigger than a single chunk', async () => { this.timeout(60 * 1000) - pull( - values([{ - path: '1.2MiB.txt', - content: values([bigFile]) - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(stringifyMh(files)).to.be.eql([expected['1.2MiB.txt']]) - done() - }) - ) + + const files = [] + + for await (const file of importer([{ + path: '1.2MiB.txt', + content: bigFile + }], ipld, options)) { + files.push(file) + } + + expect(stringifyMh(files)).to.be.eql([expected['1.2MiB.txt']]) }) - it('file bigger than a single chunk inside a dir', function (done) { + it('file bigger than a single chunk inside a dir', async () => { this.timeout(60 * 1000) - pull( - values([{ - path: 'foo-big/1.2MiB.txt', - content: values([bigFile]) - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - - expect(stringifyMh(files)).to.be.eql([ - expected['foo-big/1.2MiB.txt'], - expected['foo-big'] - ]) - - done() - }) - ) + + const files = [] + + for await (const file of importer([{ + path: 'foo-big/1.2MiB.txt', + content: bigFile + }], ipld, options)) { + files.push(file) + } + + expect(stringifyMh(files)).to.deep.equal([ + expected['foo-big/1.2MiB.txt'], + expected['foo-big'] + ]) }) - it('empty directory', (done) => { - pull( - values([{ - path: 'empty-dir' - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() + it('empty directory', async () => { + const files = [] - expect(stringifyMh(files)).to.be.eql([expected['empty-dir']]) + for await (const file of importer([{ + path: 'empty-dir' + }], ipld, options)) { + files.push(file) + } - done() - }) - ) + expect(stringifyMh(files)).to.be.eql([expected['empty-dir']]) }) - it('directory with files', (done) => { - pull( - values([{ - path: 'pim/200Bytes.txt', - content: values([smallFile]) - }, { - path: 'pim/1.2MiB.txt', - content: values([bigFile]) - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - - expect(stringifyMh(files)).be.eql([ - expected['pim/200Bytes.txt'], - expected['pim/1.2MiB.txt'], - expected.pim] - ) - - done() - }) + it('directory with files', async () => { + const files = [] + + for await (const file of importer([{ + path: 'pim/200Bytes.txt', + content: smallFile + }, { + path: 'pim/1.2MiB.txt', + content: bigFile + }], ipld, options)) { + files.push(file) + } + + expect(stringifyMh(files)).be.eql([ + expected['pim/200Bytes.txt'], + expected['pim/1.2MiB.txt'], + expected.pim] ) }) - it('nested directory (2 levels deep)', (done) => { - pull( - values([{ - path: 'pam/pum/200Bytes.txt', - content: values([smallFile]) - }, { - path: 'pam/pum/1.2MiB.txt', - content: values([bigFile]) - }, { - path: 'pam/1.2MiB.txt', - content: values([bigFile]) - }]), - importer(ipld, options), - collect((err, files) => { - expect(err).to.not.exist() - - // need to sort as due to parallel storage the order - // can vary - stringifyMh(files).forEach(eachFile) - - done() - }) - ) + it('nested directory (2 levels deep)', async () => { + const files = [] + + for await (const file of importer([{ + path: 'pam/pum/200Bytes.txt', + content: smallFile + }, { + path: 'pam/pum/1.2MiB.txt', + content: bigFile + }, { + path: 'pam/1.2MiB.txt', + content: bigFile + }], ipld, options)) { + files.push(file) + } + + stringifyMh(files).forEach(eachFile) function eachFile (file) { if (file.path === 'pam/pum/200Bytes.txt') { - expect(file.multihash).to.be.eql(expected['200Bytes.txt'].multihash) + expect(file.cid).to.be.eql(expected['200Bytes.txt'].cid) expect(file.size).to.be.eql(expected['200Bytes.txt'].size) } if (file.path === 'pam/pum/1.2MiB.txt') { - expect(file.multihash).to.be.eql(expected['1.2MiB.txt'].multihash) + expect(file.cid).to.be.eql(expected['1.2MiB.txt'].cid) expect(file.size).to.be.eql(expected['1.2MiB.txt'].size) } if (file.path === 'pam/pum') { const dir = expected['pam/pum'] - expect(file.multihash).to.be.eql(dir.multihash) + expect(file.cid).to.be.eql(dir.cid) expect(file.size).to.be.eql(dir.size) } if (file.path === 'pam/1.2MiB.txt') { - expect(file.multihash).to.be.eql(expected['1.2MiB.txt'].multihash) + expect(file.cid).to.be.eql(expected['1.2MiB.txt'].cid) expect(file.size).to.be.eql(expected['1.2MiB.txt'].size) } if (file.path === 'pam') { const dir = expected.pam - expect(file.multihash).to.be.eql(dir.multihash) + expect(file.cid).to.be.eql(dir.cid) expect(file.size).to.be.eql(dir.size) } } }) - it('will not write to disk if passed "onlyHash" option', (done) => { + it('will not write to disk if passed "onlyHash" option', async () => { const content = String(Math.random() + Date.now()) - const inputFile = { + const files = [] + + for await (const file of importer([{ path: content + '.txt', content: Buffer.from(content) - } - - const options = { + }], ipld, { onlyHash: true + })) { + files.push(file) } - const onCollected = (err, files) => { - if (err) return done(err) + const file = files[0] + expect(file).to.exist() - const file = files[0] - expect(file).to.exist() + try { + await ipld.get(file.cid) - ipld.get(new CID(file.multihash), (err) => { - expect(err).to.exist() - done() - }) + throw new Error('No error was thrown') + } catch (err) { + expect(err.code).to.equal('ERR_NOT_FOUND') } - - pull( - values([inputFile]), - importer(ipld, options), - collect(onCollected) - ) }) - it('will call an optional progress function', (done) => { - options.progress = spy() - - pull( - values([{ - path: '1.2MiB.txt', - content: values([bigFile]) - }]), - importer(ipld, options), - collect(() => { - expect(options.progress.called).to.equal(true) - expect(options.progress.args[0][0]).to.equal(1024) - done() - }) - ) + it('will call an optional progress function', async () => { + const maxChunkSize = 2048 + + const options = { + progress: spy(), + chunkerOptions: { + maxChunkSize + } + } + + for await (const _ of importer([{ // eslint-disable-line no-unused-vars + path: '1.2MiB.txt', + content: bigFile + }], ipld, options)) { + // falala + } + + expect(options.progress.called).to.equal(true) + expect(options.progress.args[0][0]).to.equal(maxChunkSize) }) - it('will import files with CID version 1', (done) => { + it('will import files with CID version 1', async () => { const createInputFile = (path, size) => { const name = String(Math.random() + Date.now()) path = path[path.length - 1] === '/' ? path : path + '/' @@ -646,104 +568,79 @@ strategies.forEach((strategy) => { shardSplitThreshold: 3 } - const onCollected = (err, files) => { - if (err) return done(err) - - const file = files[0] - expect(file).to.exist() - - each(files, (file, cb) => { - const cid = new CID(file.multihash).toV1() - const inputFile = inputFiles.find(f => f.path === file.path) - - // Just check the intermediate directory can be retrieved - if (!inputFile) { - return ipld.get(cid, cb) - } - - // Check the imported content is correct - pull( - exporter(cid, ipld), - collect((err, nodes) => { - expect(err).to.not.exist() - pull( - nodes[0].content, - collect((err, chunks) => { - expect(err).to.not.exist() - expect(Buffer.concat(chunks)).to.deep.equal(inputFile.content) - cb() - }) - ) - }) - ) - }, done) + const files = [] + + // Pass a copy of inputFiles, since the importer mutates them + for await (const file of importer(inputFiles.map(f => Object.assign({}, f)), ipld, options)) { + files.push(file) } - pull( - // Pass a copy of inputFiles, since the importer mutates them - values(inputFiles.map(f => Object.assign({}, f))), - importer(ipld, options), - collect(onCollected) - ) + const file = files[0] + expect(file).to.exist() + + for (let i = 0; i < file.length; i++) { + const file = files[i] + + const cid = file.cid.toV1() + const inputFile = inputFiles.find(f => f.path === file.path) + + // Just check the intermediate directory can be retrieved + if (!inputFile) { + await ipld.get(cid) + } + + // Check the imported content is correct + const node = await exporter(cid, ipld) + const chunks = [] + + for await (const chunk of node.content()) { + chunks.push(chunk) + } + + expect(Buffer.concat(chunks)).to.deep.equal(inputFile.content) + } }) - it('imports file with raw leaf nodes when specified', (done) => { - checkLeafNodeTypes(ipld, { + it('imports file with raw leaf nodes when specified', async () => { + return checkLeafNodeTypes(ipld, { leafType: 'raw' - }, 'raw', done) + }, 'raw') }) - it('imports file with file leaf nodes when specified', (done) => { - checkLeafNodeTypes(ipld, { + it('imports file with file leaf nodes when specified', async () => { + return checkLeafNodeTypes(ipld, { leafType: 'file' - }, 'file', done) + }, 'file') }) - it('reduces file to single node when specified', (done) => { - checkNodeLinks(ipld, { + it('reduces file to single node when specified', async () => { + return checkNodeLinks(ipld, { reduceSingleLeafToSelf: true - }, 0, done) + }, 0) }) - it('does not reduce file to single node when overidden by options', (done) => { - checkNodeLinks(ipld, { + it('does not reduce file to single node when overidden by options', async () => { + return checkNodeLinks(ipld, { reduceSingleLeafToSelf: false - }, 1, done) + }, 1) }) - it('uses raw leaf nodes when requested', (done) => { + it('uses raw leaf nodes when requested', async () => { this.timeout(60 * 1000) - options.rawLeaves = true - - pull( - values([{ - path: '1.2MiB.txt', - content: values([bigFile]) - }]), - importer(ipld, options), - collect((error, files) => { - expect(error).to.not.exist() - - const node = files[0] - - collectLeafCids(ipld, node.multihash, (error, cids) => { - expect(error).to.be.not.ok() - - const rawNodes = cids - .filter(cid => cid.codec === 'raw') - - expect(rawNodes).to.not.be.empty() - - rawNodes - .forEach(cid => { - expect(cid.version).to.equal(1) - }) + const options = { + rawLeaves: true + } - done() - }) - }) - ) + for await (const file of importer([{ + path: '1.2MiB.txt', + content: bigFile + }], ipld, options)) { + for await (const { cid } of collectLeafCids(file.cid, ipld)) { + expect(cid.codec).to.be('raw') + expect(cid.version).to.be(1) + } + } }) }) }) diff --git a/test/with-dag-api.spec.js b/test/with-dag-api.spec.js deleted file mode 100644 index 9728647..0000000 --- a/test/with-dag-api.spec.js +++ /dev/null @@ -1,436 +0,0 @@ -/* eslint-env mocha */ -/* eslint max-nested-callbacks: ["error", 8] */ - -'use strict' - -const importer = require('./../src') - -const extend = require('deep-extend') -const chai = require('chai') -chai.use(require('dirty-chai')) -const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const empty = require('pull-stream/sources/empty') -const onEnd = require('pull-stream/sinks/on-end') -const collect = require('pull-stream/sinks/collect') -const loadFixture = require('aegir/fixtures') -const CID = require('cids') -const IPLD = require('ipld') -const inMemory = require('ipld-in-memory') - -function stringifyMh (files) { - return files.map((file) => { - file.multihash = new CID(file.multihash).toBaseEncodedString() - return file - }) -} - -const bigFile = loadFixture('test/fixtures/1.2MiB.txt') -const smallFile = loadFixture('test/fixtures/200Bytes.txt') - -const baseFiles = { - '200Bytes.txt': { - path: '200Bytes.txt', - multihash: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', - size: 211, - name: '', - leafSize: 200 - }, - '1.2MiB.txt': { - path: '1.2MiB.txt', - multihash: 'QmbPN6CXXWpejfQgnRYnMQcVYkFHEntHWqLNQjbkatYCh1', - size: 1328062, - name: '', - leafSize: 1258000 - } -} - -const strategyBaseFiles = { - flat: baseFiles, - balanced: extend({}, baseFiles, { - '1.2MiB.txt': { - multihash: 'QmeEGqUisUD2T6zU96PrZnCkHfXCGuQeGWKu4UoSuaZL3d', - size: 1335420 - } - }), - trickle: extend({}, baseFiles, { - '1.2MiB.txt': { - multihash: 'QmaiSohNUt1rBf2Lqz6ou54NHVPTbXbBoPuq9td4ekcBx4', - size: 1334599 - } - }) -} - -const strategies = [ - 'flat', - 'balanced', - 'trickle' -] - -const strategyOverrides = { - balanced: { - 'foo-big': { - path: 'foo-big', - multihash: 'QmQ1S6eEamaf4t948etp8QiYQ9avrKCogiJnPRgNkVreLv', - size: 1335478 - }, - pim: { - multihash: 'QmUpzaN4Jio2GB3HoPSRCMQD5EagdMWjSEGD4SGZXaCw7W', - size: 1335744 - }, - 'pam/pum': { - multihash: 'QmUpzaN4Jio2GB3HoPSRCMQD5EagdMWjSEGD4SGZXaCw7W', - size: 1335744 - }, - pam: { - multihash: 'QmVoVD4fEWFLJLjvRCg4bGrziFhgECiaezp79AUfhuLgno', - size: 2671269 - } - }, - trickle: { - 'foo-big': { - path: 'foo-big', - multihash: 'QmPh6KSS7ghTqzgWhaoCiLoHFPF7HGqUxx7q9vcM5HUN4U', - size: 1334657 - }, - pim: { - multihash: 'QmPAn3G2x2nrq4A1fu2XUpwWtpqG4D1YXFDrU615NHvJbr', - size: 1334923 - }, - 'pam/pum': { - multihash: 'QmPAn3G2x2nrq4A1fu2XUpwWtpqG4D1YXFDrU615NHvJbr', - size: 1334923 - }, - pam: { - multihash: 'QmZTJah1xpG9X33ZsPtDEi1tYSHGDqQMRHsGV5xKzAR2j4', - size: 2669627 - } - } - -} - -describe('with dag-api', function () { - strategies.forEach(strategy => { - const baseFiles = strategyBaseFiles[strategy] - const defaultResults = extend({}, baseFiles, { - 'foo/bar/200Bytes.txt': extend({}, baseFiles['200Bytes.txt'], { - path: 'foo/bar/200Bytes.txt' - }), - foo: { - path: 'foo', - multihash: 'QmQrb6KKWGo8w7zKfx2JksptY6wN7B2ysSBdKZr4xMU36d', - size: 320 - }, - 'foo/bar': { - path: 'foo/bar', - multihash: 'Qmf5BQbTUyUAvd6Ewct83GYGnE1F6btiC3acLhR8MDxgkD', - size: 270 - }, - 'foo-big/1.2MiB.txt': extend({}, baseFiles['1.2MiB.txt'], { - path: 'foo-big/1.2MiB.txt' - }), - 'foo-big': { - path: 'foo-big', - multihash: 'Qma6JU3FoXU9eAzgomtmYPjzFBwVc2rRbECQpmHFiA98CJ', - size: 1328120 - }, - 'pim/200Bytes.txt': extend({}, baseFiles['200Bytes.txt'], { - path: 'pim/200Bytes.txt' - }), - 'pim/1.2MiB.txt': extend({}, baseFiles['1.2MiB.txt'], { - path: 'pim/1.2MiB.txt' - }), - pim: { - path: 'pim', - multihash: 'QmNk8VPGb3fkAQgoxctXo4Wmnr4PayFTASy4MiVXTtXqiA', - size: 1328386 - }, - 'empty-dir': { - path: 'empty-dir', - multihash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn', - size: 4 - }, - 'pam/pum': { - multihash: 'QmNk8VPGb3fkAQgoxctXo4Wmnr4PayFTASy4MiVXTtXqiA', - size: 1328386 - }, - pam: { - multihash: 'QmPAixYTaYnPe795fcWcuRpo6tfwHgRKNiBHpMzoomDVN6', - size: 2656553 - } - }, strategyOverrides[strategy]) - - const expected = extend({}, defaultResults, strategies[strategy]) - - describe('importer: ' + strategy, function () { - this.timeout(50 * 1000) - - let dag - - const options = { - strategy: strategy, - maxChildrenPerNode: 10, - chunkerOptions: { - maxChunkSize: 1024 - } - } - - before(function (done) { - inMemory(IPLD, (err, resolver) => { - if (err) { - return done(err) - } - - dag = resolver - - done() - }) - }) - - it('fails on bad input', (done) => { - pull( - values([{ - path: '200Bytes.txt', - content: 'banana' - }]), - importer(dag, options), - onEnd((err) => { - expect(err).to.exist() - done() - }) - ) - }) - - it('doesn\'t yield anything on empty source', (done) => { - pull( - empty(), - importer(dag, options), - collect((err, nodes) => { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(0) - done() - })) - }) - - it('doesn\'t yield anything on empty file', (done) => { - pull( - values([{ - path: 'emptyfile', - content: empty() - }]), - importer(dag, options), - collect((err, nodes) => { - expect(err).to.not.exist() - expect(nodes.length).to.be.eql(1) - // always yield empty node - expect(new CID(nodes[0].multihash).toBaseEncodedString()).to.be.eql('QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH') - done() - })) - }) - - it('fails on more than one root', (done) => { - pull( - values([ - { - path: '/beep/200Bytes.txt', - content: values([smallFile]) - }, - { - path: '/boop/200Bytes.txt', - content: values([smallFile]) - } - ]), - importer(dag, options), - onEnd((err) => { - expect(err).to.exist() - expect(err.message).to.be.eql('detected more than one root') - done() - }) - ) - }) - - it('small file (smaller than a chunk)', (done) => { - pull( - values([{ - path: '200Bytes.txt', - content: values([smallFile]) - }]), - importer(dag, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt']]) - done() - }) - ) - }) - - it('small file as buffer (smaller than a chunk)', (done) => { - pull( - values([{ - path: '200Bytes.txt', - content: smallFile - }]), - importer(dag, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt']]) - done() - }) - ) - }) - - it('small file (smaller than a chunk) inside a dir', (done) => { - pull( - values([{ - path: 'foo/bar/200Bytes.txt', - content: values([smallFile]) - }]), - importer(dag, options), - collect(collected) - ) - - function collected (err, files) { - expect(err).to.not.exist() - expect(files.length).to.equal(3) - stringifyMh(files).forEach((file) => { - if (file.path === 'foo/bar/200Bytes.txt') { - expect(file).to.be.eql(expected['foo/bar/200Bytes.txt']) - } - if (file.path === 'foo') { - expect(file).to.be.eql(expected.foo) - } - if (file.path === 'foo/bar') { - expect(file).to.be.eql(expected['foo/bar']) - } - }) - done() - } - }) - - it('file bigger than a single chunk', (done) => { - pull( - values([{ - path: '1.2MiB.txt', - content: values([bigFile]) - }]), - importer(dag, options), - collect((err, files) => { - expect(err).to.not.exist() - expect(stringifyMh(files)).to.be.eql([expected['1.2MiB.txt']]) - done() - }) - ) - }) - - it('file bigger than a single chunk inside a dir', (done) => { - pull( - values([{ - path: 'foo-big/1.2MiB.txt', - content: values([bigFile]) - }]), - importer(dag, options), - collect((err, files) => { - expect(err).to.not.exist() - - expect(stringifyMh(files)).to.be.eql([ - expected['foo-big/1.2MiB.txt'], - expected['foo-big'] - ]) - - done() - }) - ) - }) - - it('empty directory', (done) => { - pull( - values([{ - path: 'empty-dir' - }]), - importer(dag, options), - collect((err, files) => { - expect(err).to.not.exist() - - expect(stringifyMh(files)).to.be.eql([expected['empty-dir']]) - - done() - }) - ) - }) - - it('directory with files', (done) => { - pull( - values([{ - path: 'pim/200Bytes.txt', - content: values([smallFile]) - }, { - path: 'pim/1.2MiB.txt', - content: values([bigFile]) - }]), - importer(dag, options), - collect((err, files) => { - expect(err).to.not.exist() - - expect(stringifyMh(files)).be.eql([ - expected['pim/200Bytes.txt'], - expected['pim/1.2MiB.txt'], - expected.pim] - ) - - done() - }) - ) - }) - - it('nested directory (2 levels deep)', (done) => { - pull( - values([{ - path: 'pam/pum/200Bytes.txt', - content: values([smallFile]) - }, { - path: 'pam/pum/1.2MiB.txt', - content: values([bigFile]) - }, { - path: 'pam/1.2MiB.txt', - content: values([bigFile]) - }]), - importer(dag, options), - collect((err, files) => { - expect(err).to.not.exist() - - // need to sort as due to parallel storage the order can vary - stringifyMh(files).forEach(eachFile) - - done() - }) - ) - - function eachFile (file) { - if (file.path === 'pam/pum/200Bytes.txt') { - expect(file.cid).to.be.eql(expected['200Bytes.txt'].cid) - expect(file.size).to.be.eql(expected['200Bytes.txt'].size) - } - if (file.path === 'pam/pum/1.2MiB.txt') { - expect(file.cid).to.be.eql(expected['1.2MiB.txt'].cid) - expect(file.size).to.be.eql(expected['1.2MiB.txt'].size) - } - if (file.path === 'pam/pum') { - const dir = expected['pam/pum'] - expect(file.cid).to.be.eql(dir.cid) - expect(file.size).to.be.eql(dir.size) - } - if (file.path === 'pam/1.2MiB.txt') { - expect(file.cid).to.be.eql(expected['1.2MiB.txt'].cid) - expect(file.size).to.be.eql(expected['1.2MiB.txt'].size) - } - if (file.path === 'pam') { - const dir = expected.pam - expect(file.cid).to.be.eql(dir.cid) - expect(file.size).to.be.eql(dir.size) - } - } - }) - }) - }) -}) From 5ff273993185bb7441a0334cf446a34a78e88544 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 May 2019 20:54:29 +0100 Subject: [PATCH 2/4] chore: update ipld formats --- README.md | 100 ++-- package.json | 35 +- src/builder/balanced/balanced-reducer.js | 60 --- src/builder/balanced/index.js | 12 - src/builder/builder.js | 186 ------- src/builder/create-build-stream.js | 20 - src/builder/flat/index.js | 39 -- src/builder/index.js | 32 -- src/builder/reduce.js | 51 -- src/builder/trickle/index.js | 13 - src/builder/trickle/trickle-reducer.js | 147 ------ src/chunker/fixed-size.js | 59 +-- src/chunker/index.js | 12 +- src/chunker/rabin.js | 37 +- src/dag-builder/dir.js | 23 + src/dag-builder/file/balanced.js | 23 + src/dag-builder/file/flat.js | 13 + src/dag-builder/file/index.js | 134 ++++++ src/dag-builder/file/trickle.js | 106 ++++ src/dag-builder/index.js | 44 ++ src/dag-builder/validate-chunks.js | 22 + src/dir-flat.js | 86 ++++ src/dir-sharded.js | 152 ++++++ src/{importer => }/dir.js | 4 +- src/flat-to-shard.js | 47 ++ src/importer/dir-flat.js | 84 ---- src/importer/dir-sharded.js | 192 -------- src/importer/flat-to-shard.js | 74 --- src/importer/index.js | 117 ----- src/importer/tree-builder.js | 215 --------- src/index.js | 121 ++++- src/tree-builder.js | 92 ++++ src/utils/persist.js | 53 +- test/builder-balanced.spec.js | 20 +- test/builder-dir-sharding.spec.js | 20 +- test/builder-flat.spec.js | 20 +- test/builder-only-hash.spec.js | 37 +- test/builder-trickle-dag.spec.js | 355 ++++++++++---- test/builder.spec.js | 48 +- test/chunker-fixed-size.spec.js | 32 +- test/chunker-rabin-browser.spec.js | 30 +- test/chunker-rabin.spec.js | 30 +- test/hash-parity-with-go-ipfs.spec.js | 6 +- test/helpers/collect-leaf-cids.js | 4 +- .../finite-pseudorandom-byte-stream.js | 41 +- test/helpers/random-byte-stream.js | 16 +- test/import-export-nested-dir.spec.js | 86 ++-- test/import-export.spec.js | 5 +- test/importer-flush.spec.js | 204 -------- test/importer.spec.js | 454 +++++++++--------- 50 files changed, 1629 insertions(+), 2184 deletions(-) delete mode 100644 src/builder/balanced/balanced-reducer.js delete mode 100644 src/builder/balanced/index.js delete mode 100644 src/builder/builder.js delete mode 100644 src/builder/create-build-stream.js delete mode 100644 src/builder/flat/index.js delete mode 100644 src/builder/index.js delete mode 100644 src/builder/reduce.js delete mode 100644 src/builder/trickle/index.js delete mode 100644 src/builder/trickle/trickle-reducer.js create mode 100644 src/dag-builder/dir.js create mode 100644 src/dag-builder/file/balanced.js create mode 100644 src/dag-builder/file/flat.js create mode 100644 src/dag-builder/file/index.js create mode 100644 src/dag-builder/file/trickle.js create mode 100644 src/dag-builder/index.js create mode 100644 src/dag-builder/validate-chunks.js create mode 100644 src/dir-flat.js create mode 100644 src/dir-sharded.js rename src/{importer => }/dir.js (53%) create mode 100644 src/flat-to-shard.js delete mode 100644 src/importer/dir-flat.js delete mode 100644 src/importer/dir-sharded.js delete mode 100644 src/importer/flat-to-shard.js delete mode 100644 src/importer/index.js delete mode 100644 src/importer/tree-builder.js create mode 100644 src/tree-builder.js delete mode 100644 test/importer-flush.spec.js diff --git a/README.md b/README.md index 7076b7d..4824e3f 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,16 @@ ## Table of Contents -- [Install](#install) -- [Usage](#usage) - - [Example](#example) - - [API](#api) - - [const add = importer(dag, options)](#const-import--importerdag--options) -- [Contribute](#contribute) -- [License](#license) +- [ipfs-unixfs-importer](#ipfs-unixfs-importer) + - [Lead Maintainer](#lead-maintainer) + - [Table of Contents](#table-of-contents) + - [Install](#install) + - [Usage](#usage) + - [Example](#example) + - [API](#api) + - [const import = importer(source, ipld [, options])](#const-import--importersource-ipld--options) + - [Contribute](#contribute) + - [License](#license) ## Install @@ -50,51 +53,46 @@ And write the importing logic: ```js const importer = require('ipfs-unixfs-importer') -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') // Import path /tmp/foo/bar -pull( - values([{ - path: '/tmp/foo/bar', - content: fs.createReadStream(file) - }, { - path: '/tmp/foo/quxx', - content: fs.createReadStream(file2) - } - }]), - - // You need to create and pass an ipld-resolve instance - // https://github.com/ipld/js-ipld-resolver - importer(, ), - - // Handle the error and do something with the results - collect((err, files) => { - console.info(files) - }) -) +const source = [{ + path: '/tmp/foo/bar', + content: fs.createReadStream(file) +}, { + path: '/tmp/foo/quxx', + content: fs.createReadStream(file2) +}] + +// You need to create and pass an ipld-resolve instance +// https://github.com/ipld/js-ipld-resolver +for await (const entry of importer(source, ipld, options)) { + console.info(entry) +} ``` -When run, the stat of DAGNode is outputted for each file on data event until the root: +When run, metadata about DAGNodes in the created tree is printed until the root: ```js -{ multihash: , - size: 39243, - path: '/tmp/foo/bar' } - -{ multihash: , - size: 59843, - path: '/tmp/foo/quxx' } - -{ multihash: , - size: 93242, - path: '/tmp/foo' } - -{ multihash: , - size: 94234, - path: '/tmp' } - +{ + cid: CID, // see https://github.com/multiformats/js-cid + path: '/tmp/foo/bar', + unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs +} +{ + cid: CID, // see https://github.com/multiformats/js-cid + path: '/tmp/foo/quxx', + unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs +} +{ + cid: CID, // see https://github.com/multiformats/js-cid + path: '/tmp/foo', + unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs +} +{ + cid: CID, // see https://github.com/multiformats/js-cid + path: '/tmp', + unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs +} ``` #### API @@ -103,20 +101,20 @@ When run, the stat of DAGNode is outputted for each file on data event until the const importer = require('ipfs-unixfs-importer') ``` -#### const import = importer(dag [, options]) +#### const import = importer(source, ipld [, options]) -The `import` object is a duplex pull stream that takes objects of the form: +The `import` function returns an async iterator takes a source async iterator that yields objects of the form: ```js { path: 'a name', - content: (Buffer, pull-stream emitting Buffers or a Readable stream) + content: (Buffer or iterator emitting Buffers) } ``` `import` will output file info objects as files get stored in IPFS. When stats on a node are emitted they are guaranteed to have been written. -`dag` is an instance of the [`IPLD Resolver`](https://github.com/ipld/js-ipld-resolver) or the [`js-ipfs` `dag api`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md) +`ipld` is an instance of the [`IPLD Resolver`](https://github.com/ipld/js-ipld-resolver) or the [`js-ipfs` `dag api`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md) The input's file paths and directory structure will be preserved in the [`dag-pb`](https://github.com/ipld/js-ipld-dag-pb) created nodes. @@ -148,10 +146,8 @@ The input's file paths and directory structure will be preserved in the [`dag-pb - `rawLeaves` (boolean, defaults to false): When a file would span multiple DAGNodes, if this is true the leaf nodes will not be wrapped in `UnixFS` protobufs and will instead contain the raw file bytes - `leafType` (string, defaults to `'file'`) what type of UnixFS node leaves should be - can be `'file'` or `'raw'` (ignored when `rawLeaves` is `true`) -[dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md [ipld-resolver instance]: https://github.com/ipld/js-ipld-resolver [UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs -[pull-stream]: https://www.npmjs.com/package/pull-stream ## Contribute diff --git a/package.json b/package.json index ffd5ea6..2cbefac 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "release": "aegir release", "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", - "coverage": "aegir coverage", + "coverage": "nyc -s npm run test:node && nyc report --reporter=html", "dep-check": "aegir dep-check" }, "repository": { @@ -39,44 +39,39 @@ "homepage": "https://github.com/ipfs/js-ipfs-unixfs-importer#readme", "devDependencies": { "aegir": "^18.0.2", - "async-iterator-all": "0.0.2", - "async-iterator-buffer-stream": "~0.0.1", - "async-iterator-first": "0.0.2", - "async-iterator-last": "0.0.2", + "async-iterator-all": "^1.0.0", + "async-iterator-buffer-stream": "^1.0.0", + "async-iterator-first": "^1.0.0", + "async-iterator-last": "^1.0.0", "chai": "^4.2.0", - "cids": "~0.5.5", + "cids": "~0.7.1", "detect-node": "^2.0.4", "dirty-chai": "^2.0.1", - "ipfs-unixfs-exporter": "~0.36.1", - "ipld": "~0.22.0", + "ipfs-unixfs-exporter": "0.36.1", + "ipld": "~0.24.0", "ipld-in-memory": "^2.0.0", "multihashes": "~0.4.14", + "nyc": "^14.0.0", "pull-generate": "^2.2.0", "pull-traverse": "^1.0.3", "sinon": "^7.1.0" }, "dependencies": { "async": "^2.6.1", + "async-iterator-batch": "~0.0.1", "async-iterator-to-pull-stream": "^1.3.0", "bl": "^3.0.0", "deep-extend": "~0.6.0", "err-code": "^1.1.2", "hamt-sharding": "~0.0.2", "ipfs-unixfs": "~0.1.16", - "ipld-dag-pb": "~0.15.2", + "ipld-dag-pb": "~0.17.0", "left-pad": "^1.3.0", "multicodec": "~0.5.1", - "multihashing-async": "~0.5.1", - "pull-batch": "^1.0.0", - "pull-pair": "^1.1.0", - "pull-paramap": "^1.2.2", - "pull-pause": "0.0.2", - "pull-pushable": "^2.2.0", - "pull-stream": "^3.6.9", - "pull-stream-to-async-iterator": "^1.0.1", - "pull-through": "^1.0.18", - "pull-write": "^1.1.4", - "stream-to-pull-stream": "^1.7.2" + "multihashing-async": "~0.7.0", + "stream-to-async-iterator": "~0.2.0", + "stream-to-pull-stream": "^1.7.2", + "superstruct": "~0.6.1" }, "optionalDependencies": { "rabin": "^1.6.0" diff --git a/src/builder/balanced/balanced-reducer.js b/src/builder/balanced/balanced-reducer.js deleted file mode 100644 index 3781eec..0000000 --- a/src/builder/balanced/balanced-reducer.js +++ /dev/null @@ -1,60 +0,0 @@ -'use strict' - -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const asyncMap = require('pull-stream/throughs/async-map') -const collect = require('pull-stream/sinks/collect') -const pushable = require('pull-pushable') -const pullPair = require('pull-pair') -const batch = require('pull-batch') - -module.exports = function balancedReduceToRoot (reduce, options) { - const pair = pullPair() - const source = pair.source - - const result = pushable() - - reduceToParents(source, (err, roots) => { - if (err) { - result.end(err) - return // early - } - if (roots.length === 1) { - result.push(roots[0]) - result.end() - } else if (roots.length > 1) { - result.end(new Error('expected a maximum of 1 roots and got ' + roots.length)) - } else { - result.end() - } - }) - - function reduceToParents (_chunks, callback) { - let chunks = _chunks - if (Array.isArray(chunks)) { - chunks = values(chunks) - } - - pull( - chunks, - batch(options.maxChildrenPerNode), - asyncMap(reduce), - collect(reduced) - ) - - function reduced (err, roots) { - if (err) { - callback(err) - } else if (roots.length > 1) { - reduceToParents(roots, callback) - } else { - callback(null, roots) - } - } - } - - return { - sink: pair.sink, - source: result - } -} diff --git a/src/builder/balanced/index.js b/src/builder/balanced/index.js deleted file mode 100644 index cc43d8b..0000000 --- a/src/builder/balanced/index.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict' - -const balancedReducer = require('./balanced-reducer') - -const defaultOptions = { - maxChildrenPerNode: 174 -} - -module.exports = function (reduce, _options) { - const options = Object.assign({}, defaultOptions, _options) - return balancedReducer(reduce, options) -} diff --git a/src/builder/builder.js b/src/builder/builder.js deleted file mode 100644 index c97ba9d..0000000 --- a/src/builder/builder.js +++ /dev/null @@ -1,186 +0,0 @@ -'use strict' - -const extend = require('deep-extend') -const UnixFS = require('ipfs-unixfs') -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') -const through = require('pull-stream/throughs/through') -const pullThrough = require('pull-through') -const parallel = require('async/parallel') -const waterfall = require('async/waterfall') -const paraMap = require('pull-paramap') -const persist = require('../utils/persist') -const reduce = require('./reduce') -const { - DAGNode -} = require('ipld-dag-pb') -const errCode = require('err-code') - -const defaultOptions = { - chunkerOptions: { - maxChunkSize: 262144, - avgChunkSize: 262144 - }, - rawLeaves: false, - hashAlg: 'sha2-256', - leafType: 'file', - cidVersion: 0, - progress: () => {} -} - -module.exports = function builder (createChunker, ipld, createReducer, _options) { - const options = extend({}, defaultOptions, _options) - options.progress = typeof options.progress === 'function' ? options.progress : defaultOptions.progress - - return function (source) { - return function (items, cb) { - parallel(items.map((item) => (cb) => { - if (!item.content) { - // item is a directory - return createAndStoreDir(item, (err, node) => { - if (err) { - return cb(err) - } - if (node) { - source.push(node) - } - cb() - }) - } - - // item is a file - createAndStoreFile(item, (err, node) => { - if (err) { - return cb(err) - } - if (node) { - source.push(node) - } - cb() - }) - }), cb) - } - } - - function createAndStoreDir (item, callback) { - // 1. create the empty dir dag node - // 2. write it to the dag store - - const d = new UnixFS('directory') - - waterfall([ - (cb) => DAGNode.create(d.marshal(), [], cb), - (node, cb) => persist(node, ipld, options, cb) - ], (err, result) => { - if (err) { - return callback(err) - } - - callback(null, { - path: item.path, - cid: result.cid, - size: result.node.size - }) - }) - } - - function createAndStoreFile (file, callback) { - if (Buffer.isBuffer(file.content)) { - file.content = values([file.content]) - } - - if (typeof file.content !== 'function') { - return callback(errCode(new Error('invalid content'), 'EINVALIDCONTENT')) - } - - const reducer = createReducer(reduce(file, ipld, options), options) - let chunker - - try { - chunker = createChunker(options.chunkerOptions) - } catch (error) { - return callback(error) - } - - let previous - let count = 0 - - pull( - file.content, - chunker, - through(buffer => { - options.progress(buffer.length) - }), - paraMap((buffer, callback) => { - waterfall([ - (cb) => { - if (options.rawLeaves) { - return cb(null, { - size: buffer.length, - leafSize: buffer.length, - data: buffer - }) - } - - const file = new UnixFS(options.leafType, buffer) - - DAGNode.create(file.marshal(), [], (err, node) => { - if (err) { - return cb(err) - } - - cb(null, { - size: node.size, - leafSize: file.fileSize(), - data: node - }) - }) - }, - (leaf, cb) => { - persist(leaf.data, ipld, options, (error, results) => { - if (error) { - return cb(error) - } - - cb(null, { - size: leaf.size, - leafSize: leaf.leafSize, - data: results.node, - cid: results.cid, - path: leaf.path, - name: '' - }) - }) - } - ], callback) - }), - pullThrough( // mark as single node if only one single node - function onData (data) { - count++ - if (previous) { - this.queue(previous) - } - previous = data - }, - function ended () { - if (previous) { - if (count === 1) { - previous.single = true - } - this.queue(previous) - } - this.queue(null) - } - ), - reducer, - collect((err, roots) => { - if (err) { - callback(err) - } else { - callback(null, roots[0]) - } - }) - ) - } -} diff --git a/src/builder/create-build-stream.js b/src/builder/create-build-stream.js deleted file mode 100644 index 8d9d414..0000000 --- a/src/builder/create-build-stream.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict' - -const pullPushable = require('pull-pushable') -const pullWrite = require('pull-write') - -module.exports = function createBuildStream (createStrategy, _ipld, options) { - const source = pullPushable() - - const sink = pullWrite( - createStrategy(source), - null, - options.highWaterMark, - (err) => source.end(err) - ) - - return { - source: source, - sink: sink - } -} diff --git a/src/builder/flat/index.js b/src/builder/flat/index.js deleted file mode 100644 index 324f970..0000000 --- a/src/builder/flat/index.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict' - -const pull = require('pull-stream/pull') -const asyncMap = require('pull-stream/throughs/async-map') -const collect = require('pull-stream/sinks/collect') -const pushable = require('pull-pushable') -const pullPair = require('pull-pair') -const batch = require('pull-batch') - -module.exports = function (reduce, options) { - const pair = pullPair() - const source = pair.source - const result = pushable() - - pull( - source, - batch(Infinity), - asyncMap(reduce), - collect((err, roots) => { - if (err) { - result.end(err) - return // early - } - if (roots.length === 1) { - result.push(roots[0]) - result.end() - } else if (roots.length > 1) { - result.end(new Error('expected a maximum of 1 roots and got ' + roots.length)) - } else { - result.end() - } - }) - ) - - return { - sink: pair.sink, - source: result - } -} diff --git a/src/builder/index.js b/src/builder/index.js deleted file mode 100644 index b494840..0000000 --- a/src/builder/index.js +++ /dev/null @@ -1,32 +0,0 @@ -'use strict' - -const assert = require('assert') -const createBuildStream = require('./create-build-stream') -const Builder = require('./builder') - -const reducers = { - flat: require('./flat'), - balanced: require('./balanced'), - trickle: require('./trickle') -} - -const defaultOptions = { - strategy: 'balanced', - highWaterMark: 100, - reduceSingleLeafToSelf: true -} - -module.exports = function (Chunker, ipld, _options) { - assert(Chunker, 'Missing chunker creator function') - assert(ipld, 'Missing IPLD') - - const options = Object.assign({}, defaultOptions, _options) - - const strategyName = options.strategy - const reducer = reducers[strategyName] - assert(reducer, 'Unknown importer build strategy name: ' + strategyName) - - const createStrategy = Builder(Chunker, ipld, reducer, options) - - return createBuildStream(createStrategy, ipld, options) -} diff --git a/src/builder/reduce.js b/src/builder/reduce.js deleted file mode 100644 index 99b9d55..0000000 --- a/src/builder/reduce.js +++ /dev/null @@ -1,51 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') -const dagPB = require('ipld-dag-pb') -const UnixFS = require('ipfs-unixfs') -const persist = require('../utils/persist') - -const DAGLink = dagPB.DAGLink -const DAGNode = dagPB.DAGNode - -module.exports = function reduce (file, ipld, options) { - return function (leaves, callback) { - if (leaves.length === 1 && leaves[0].single && options.reduceSingleLeafToSelf) { - const leaf = leaves[0] - - return callback(null, { - size: leaf.size, - leafSize: leaf.leafSize, - cid: leaf.cid, - path: file.path, - name: leaf.name - }) - } - - // create a parent node and add all the leaves - const f = new UnixFS('file') - - const links = leaves.map((leaf) => { - f.addBlockSize(leaf.leafSize) - - return new DAGLink(leaf.name, leaf.size, leaf.cid) - }) - - waterfall([ - (cb) => DAGNode.create(f.marshal(), links, cb), - (node, cb) => persist(node, ipld, options, cb) - ], (error, result) => { - if (error) { - return callback(error) - } - - callback(null, { - size: result.node.size, - leafSize: f.fileSize(), - cid: result.cid, - path: file.path, - name: '' - }) - }) - } -} diff --git a/src/builder/trickle/index.js b/src/builder/trickle/index.js deleted file mode 100644 index 1293148..0000000 --- a/src/builder/trickle/index.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict' - -const trickleReducer = require('./trickle-reducer') - -const defaultOptions = { - maxChildrenPerNode: 174, - layerRepeat: 4 -} - -module.exports = function (reduce, _options) { - const options = Object.assign({}, defaultOptions, _options) - return trickleReducer(reduce, options) -} diff --git a/src/builder/trickle/trickle-reducer.js b/src/builder/trickle/trickle-reducer.js deleted file mode 100644 index 39fe9c2..0000000 --- a/src/builder/trickle/trickle-reducer.js +++ /dev/null @@ -1,147 +0,0 @@ -'use strict' - -const pull = require('pull-stream/pull') -const asyncMap = require('pull-stream/throughs/async-map') -const collect = require('pull-stream/sinks/collect') -const pushable = require('pull-pushable') -const batch = require('pull-batch') -const pullPair = require('pull-pair') -const through = require('pull-through') -const pullWrite = require('pull-write') -const pause = require('pull-pause') - -module.exports = function trickleReduceToRoot (reduce, options) { - const pair = pullPair() - const result = pushable() - const pausable = pause(() => {}) - let pendingResumes = 0 - - pull( - pair.source, - pausable, - trickle(0, -1), - batch(Infinity), - asyncMap(reduce), - collect((err, roots) => { - if (err) { - result.end(err) - } else { - if (roots.length === 1) { - result.push(roots[0]) - result.end() - } else if (roots.length > 1) { - result.end(new Error('expected a maximum of 1 roots and got ' + roots.length)) - } else { - result.end() - } - } - }) - ) - - return { - sink: pair.sink, - source: result - } - - function trickle (indent, maxDepth) { - let iteration = 0 - let depth = 0 - let deeper - let aborting = false - - const result = pushable() - - return { - source: result, - sink: pullWrite(write, null, 1, end) - } - - function write (nodes, callback) { - let ended = false - const node = nodes[0] - - if (depth && !deeper) { - deeper = pushable() - - pull( - deeper, - trickle(indent + 1, depth - 1), - through( - function (d) { - this.queue(d) - }, - function (err) { - if (err) { - this.emit('error', err) - return // early - } - if (!ended) { - ended = true - pendingResumes++ - pausable.pause() - } - this.queue(null) - } - ), - batch(Infinity), - asyncMap(reduce), - collect((err, nodes) => { - pendingResumes-- - if (err) { - result.end(err) - return - } - nodes.forEach(node => { - result.push(node) - }) - iterate() - }) - ) - } - - if (deeper) { - deeper.push(node) - } else { - result.push(node) - iterate() - } - - callback() - } - - function iterate () { - deeper = null - iteration++ - if ((depth === 0 && iteration === options.maxChildrenPerNode) || - (depth > 0 && iteration === options.layerRepeat)) { - iteration = 0 - depth++ - } - - if ((!aborting && maxDepth >= 0 && depth > maxDepth) || - (aborting && !pendingResumes)) { - aborting = true - result.end() - } - - if (!pendingResumes) { - pausable.resume() - } - } - - function end (err) { - if (err) { - result.end(err) - return - } - if (deeper) { - if (!aborting) { - aborting = true - deeper.end() - } - } else { - result.end() - } - } - } -} diff --git a/src/chunker/fixed-size.js b/src/chunker/fixed-size.js index 88b07f7..48925af 100644 --- a/src/chunker/fixed-size.js +++ b/src/chunker/fixed-size.js @@ -1,50 +1,39 @@ 'use strict' const BufferList = require('bl') -const through = require('pull-through') -module.exports = (options) => { - let maxSize = (typeof options === 'number') ? options : options.maxChunkSize +module.exports = async function * fixedSizeChunker (source, options) { let bl = new BufferList() let currentLength = 0 let emitted = false + const maxChunkSize = options.maxChunkSize - return through( - function onData (buffer) { - bl.append(buffer) + for await (const buffer of source) { + bl.append(buffer) - currentLength += buffer.length + currentLength += buffer.length - while (currentLength >= maxSize) { - this.queue(bl.slice(0, maxSize)) + while (currentLength >= maxChunkSize) { + yield bl.slice(0, maxChunkSize) + emitted = true - emitted = true + // throw away consumed bytes + if (maxChunkSize === bl.length) { + bl = new BufferList() + currentLength = 0 + } else { + const newBl = new BufferList() + newBl.append(bl.shallowSlice(maxChunkSize)) + bl = newBl - // throw away consumed bytes - if (maxSize === bl.length) { - bl = new BufferList() - currentLength = 0 - } else { - const newBl = new BufferList() - newBl.append(bl.shallowSlice(maxSize)) - bl = newBl - - // update our offset - currentLength -= maxSize - } - } - }, - function onEnd () { - if (currentLength) { - this.queue(bl.slice(0, currentLength)) - emitted = true - } - - if (!emitted) { - this.queue(Buffer.alloc(0)) + // update our offset + currentLength -= maxChunkSize } - - this.queue(null) } - ) + } + + if (!emitted || currentLength) { + // return any remaining bytes or an empty buffer + yield bl.slice(0, currentLength) + } } diff --git a/src/chunker/index.js b/src/chunker/index.js index e983ae8..5244679 100644 --- a/src/chunker/index.js +++ b/src/chunker/index.js @@ -1,8 +1,18 @@ 'use strict' +const errCode = require('err-code') + const chunkers = { fixed: require('../chunker/fixed-size'), rabin: require('../chunker/rabin') } -module.exports = chunkers +module.exports = (type, source, options) => { + const chunker = chunkers[type] + + if (!chunker) { + throw errCode(new Error(`Unknkown chunker named ${type}`), 'EUNKNOWNCHUNKER') + } + + return chunker(source, options) +} diff --git a/src/chunker/rabin.js b/src/chunker/rabin.js index dfe648e..8291f4c 100644 --- a/src/chunker/rabin.js +++ b/src/chunker/rabin.js @@ -1,28 +1,24 @@ 'use strict' -const toPull = require('stream-to-pull-stream') -const through = require('pull-through') +const errCode = require('err-code') let createRabin -module.exports = (options) => { +module.exports = async function * rabinChunker (source, options) { if (!createRabin) { try { createRabin = require('rabin') if (typeof createRabin !== 'function') { - throw new Error('createRabin was not a function') + throw errCode(new Error(`createRabin was not a function`), 'EUNSUPPORTED') } } catch (err) { - const error = new Error(`Rabin chunker not available, it may have failed to install or not be supported on this platform`) - - return through(function () { - this.emit('error', error) - }) + throw errCode(new Error(`Rabin chunker not available, it may have failed to install or not be supported on this platform`), 'EUNSUPPORTED') } } let min, max, avg + if (options.minChunkSize && options.maxChunkSize && options.avgChunkSize) { avg = options.avgChunkSize min = options.minChunkSize @@ -38,9 +34,26 @@ module.exports = (options) => { min: min, max: max, bits: sizepow, - window: options.window || 16, - polynomial: options.polynomial || '0x3DF305DFB2A805' + window: options.window, + polynomial: options.polynomial }) - return toPull.duplex(rabin) + // TODO: rewrite rabin using node streams v3 + for await (const chunk of source) { + rabin.buffers.append(chunk) + rabin.pending.push(chunk) + + const sizes = [] + + rabin.rabin.fingerprint(rabin.pending, sizes) + rabin.pending = [] + + for (let i = 0; i < sizes.length; i++) { + const size = sizes[i] + const buf = rabin.buffers.slice(0, size) + rabin.buffers.consume(size) + + yield buf + } + } } diff --git a/src/dag-builder/dir.js b/src/dag-builder/dir.js new file mode 100644 index 0000000..db73d6d --- /dev/null +++ b/src/dag-builder/dir.js @@ -0,0 +1,23 @@ +'use strict' + +const UnixFS = require('ipfs-unixfs') +const persist = require('../utils/persist') +const { + DAGNode +} = require('ipld-dag-pb') + +const dirBuilder = async (item, ipld, options) => { + const unixfs = new UnixFS('directory') + const node = DAGNode.create(unixfs.marshal(), []) + const cid = await persist(node, ipld, options) + let path = item.path + + return { + cid, + path, + unixfs, + node + } +} + +module.exports = dirBuilder diff --git a/src/dag-builder/file/balanced.js b/src/dag-builder/file/balanced.js new file mode 100644 index 0000000..72b3cb8 --- /dev/null +++ b/src/dag-builder/file/balanced.js @@ -0,0 +1,23 @@ +'use strict' + +const batch = require('async-iterator-batch') + +async function * balanced (source, reduce, options) { + yield await reduceToParents(source, reduce, options) +} + +async function reduceToParents (source, reduce, options) { + const roots = [] + + for await (const chunked of batch(source, options.maxChildrenPerNode)) { + roots.push(await reduce(chunked)) + } + + if (roots.length > 1) { + return reduceToParents(roots, reduce, options) + } + + return roots[0] +} + +module.exports = balanced diff --git a/src/dag-builder/file/flat.js b/src/dag-builder/file/flat.js new file mode 100644 index 0000000..c7ba75e --- /dev/null +++ b/src/dag-builder/file/flat.js @@ -0,0 +1,13 @@ +'use strict' + +const batch = require('async-iterator-batch') + +module.exports = async function * (source, reduce) { + const roots = [] + + for await (const chunk of batch(source, Infinity)) { + roots.push(await reduce(chunk)) + } + + yield roots[0] +} diff --git a/src/dag-builder/file/index.js b/src/dag-builder/file/index.js new file mode 100644 index 0000000..a4ec634 --- /dev/null +++ b/src/dag-builder/file/index.js @@ -0,0 +1,134 @@ +'use strict' + +const errCode = require('err-code') +const UnixFS = require('ipfs-unixfs') +const persist = require('../../utils/persist') +const { + DAGNode, + DAGLink +} = require('ipld-dag-pb') +const all = require('async-iterator-all') + +const dagBuilders = { + flat: require('./flat'), + balanced: require('./balanced'), + trickle: require('./trickle') +} + +async function * buildFile (source, ipld, options) { + let count = -1 + let previous + + for await (const buffer of source) { + count++ + options.progress(buffer.length) + let node + let unixfs + + let opts = { + ...options + } + + if (options.rawLeaves) { + node = buffer + + opts.codec = 'raw' + opts.cidVersion = 1 + } else { + unixfs = new UnixFS(options.leafType, buffer) + node = DAGNode.create(unixfs.marshal(), []) + } + + const cid = await persist(node, ipld, opts) + + const entry = { + cid: cid, + unixfs, + node + } + + if (count === 0) { + previous = entry + continue + } else if (count === 1) { + yield previous + previous = null + } + + yield entry + } + + if (previous) { + previous.single = true + yield previous + } +} + +const reduce = (file, ipld, options) => { + return async function (leaves) { + if (leaves.length === 1 && leaves[0].single && options.reduceSingleLeafToSelf) { + const leaf = leaves[0] + + return { + cid: leaf.cid, + path: file.path, + name: (file.path || '').split('/').pop(), + unixfs: leaf.unixfs, + node: leaf.node + } + } + + // create a parent node and add all the leaves + const f = new UnixFS('file') + + const links = await Promise.all( + leaves.map(async (leaf) => { + if (leaf.cid.codec === 'raw') { + // node is a leaf buffer + f.addBlockSize(leaf.node.length) + + return new DAGLink(leaf.name, leaf.node.length, leaf.cid) + } + + if (!leaf.unixfs.data) { + // node is an intermediate node + f.addBlockSize(leaf.unixfs.fileSize()) + } else { + // node is a unixfs 'file' leaf node + f.addBlockSize(leaf.unixfs.data.length) + } + + return new DAGLink(leaf.name, leaf.node.size, leaf.cid) + }) + ) + + const node = DAGNode.create(f.marshal(), links) + const cid = await persist(node, ipld, options) + + return { + cid, + path: file.path, + unixfs: f, + node, + size: node.size + } + } +} + +const fileBuilder = async (file, source, ipld, options) => { + const dagBuilder = dagBuilders[options.strategy] + + if (!dagBuilder) { + throw errCode(new Error(`Unknown importer build strategy name: ${options.strategy}`), 'EBADSTRATEGY') + } + + const roots = await all(dagBuilder(buildFile(source, ipld, options), reduce(file, ipld, options), options.builderOptions)) + + if (roots.length > 1) { + throw errCode(new Error('expected a maximum of 1 roots and got ' + roots.length), 'ETOOMANYROOTS') + } + + return roots[0] +} + +module.exports = fileBuilder diff --git a/src/dag-builder/file/trickle.js b/src/dag-builder/file/trickle.js new file mode 100644 index 0000000..4995606 --- /dev/null +++ b/src/dag-builder/file/trickle.js @@ -0,0 +1,106 @@ +'use strict' + +const batch = require('async-iterator-batch') + +module.exports = async function * trickleReduceToRoot (source, reduce, options) { + yield trickleStream(source, reduce, options) +} + +async function trickleStream (source, reduce, options) { + let root = { + children: [] + } + let node = root + let maxDepth = 1 + let currentDepth = 1 + let layerSize = 0 + + for await (const layer of batch(source, options.maxChildrenPerNode)) { + node.data = layer + + let parent = node.parent || root + const nextNode = { + children: [] + } + + if (currentDepth < maxDepth) { + // the current layer can't have more children + // but we can descend a layer + node.children.push(nextNode) + nextNode.parent = node + node = nextNode + currentDepth++ + } else if (parent.children.length < options.layerRepeat) { + // the current layer can have more children + parent.children.push(nextNode) + nextNode.parent = parent + node = nextNode + } else if (currentDepth === maxDepth) { + // hit the bottom of the current iteration, can we find a sibling? + parent = findNext(root, 0, maxDepth, options) + + if (parent) { + nextNode.parent = parent + parent.children.push(nextNode) + node = nextNode + } else { + if (layerSize === 0) { + maxDepth++ + } + + layerSize++ + + if (layerSize === options.layerRepeat) { + layerSize = 0 + } + + nextNode.parent = root + root.children.push(nextNode) + node = nextNode + + currentDepth = 1 + } + } + } + + // reduce to root + return walk(root, reduce) +} + +const walk = async (node, reduce) => { + let children = [] + + if (node.children.length) { + children = await Promise.all( + node.children + .filter(child => child.data) + .map(child => walk(child, reduce)) + ) + } + + return reduce(node.data.concat(children)) +} + +const findNext = (node, depth, maxDepth, options) => { + if (depth === maxDepth) { + return + } + + let nodeMatches = false + + if (node.children.length < options.layerRepeat) { + nodeMatches = true + } + + if (node.children.length) { + const childMatches = findNext(node.children[node.children.length - 1], depth + 1, maxDepth, options) + + if (childMatches) { + return childMatches + } + } + + if (nodeMatches) { + return node + } +} diff --git a/src/dag-builder/index.js b/src/dag-builder/index.js new file mode 100644 index 0000000..a3e4415 --- /dev/null +++ b/src/dag-builder/index.js @@ -0,0 +1,44 @@ +'use strict' + +const dirBuilder = require('./dir') +const fileBuilder = require('./file') +const createChunker = require('../chunker') +const validateChunks = require('./validate-chunks') + +async function * dagBuilder (source, ipld, options) { + for await (const entry of source) { + if (entry.path) { + if (entry.path.substring(0, 2) === './') { + options.wrapWithDirectory = true + } + + entry.path = entry.path + .split('/') + .filter(path => path && path !== '.') + .join('/') + } + + if (entry.content) { + let source = entry.content + + // wrap in iterator if it is array-like or not an iterator + if ((!source[Symbol.asyncIterator] && !source[Symbol.iterator]) || source.length !== undefined) { + source = { + [Symbol.asyncIterator]: async function * () { + yield entry.content + } + } + } + + const chunker = createChunker(options.chunker, validateChunks(source), options.chunkerOptions) + + // item is a file + yield fileBuilder(entry, chunker, ipld, options) + } else { + // item is a directory + yield dirBuilder(entry, ipld, options) + } + } +} + +module.exports = dagBuilder diff --git a/src/dag-builder/validate-chunks.js b/src/dag-builder/validate-chunks.js new file mode 100644 index 0000000..839461b --- /dev/null +++ b/src/dag-builder/validate-chunks.js @@ -0,0 +1,22 @@ +'use strict' + +const errCode = require('err-code') + +// make sure the content only emits buffer-a-likes +async function * validateChunks (source) { + for await (const content of source) { + if (content.length === undefined) { + throw errCode(new Error('Content was invalid'), 'EINVALIDCONTENT') + } + + if (typeof content === 'string' || content instanceof String) { + yield Buffer.from(content, 'utf8') + } else if (Array.isArray(content)) { + yield Buffer.from(content) + } else { + yield content + } + } +} + +module.exports = validateChunks diff --git a/src/dir-flat.js b/src/dir-flat.js new file mode 100644 index 0000000..c46670c --- /dev/null +++ b/src/dir-flat.js @@ -0,0 +1,86 @@ +'use strict' + +const { + DAGLink, + DAGNode +} = require('ipld-dag-pb') +const UnixFS = require('ipfs-unixfs') +const Dir = require('./dir') +const persist = require('./utils/persist') + +class DirFlat extends Dir { + constructor (props, options) { + super(props, options) + this._children = {} + } + + put (name, value) { + this.cid = undefined + this.size = undefined + this._children[name] = value + } + + get (name) { + return this._children[name] + } + + childCount () { + return Object.keys(this._children).length + } + + directChildrenCount () { + return this.childCount() + } + + onlyChild () { + return this._children[Object.keys(this._children)[0]] + } + + * eachChildSeries () { + const keys = Object.keys(this._children) + + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + + yield { + key: key, + child: this._children[key] + } + } + } + + async * flush (path, ipld) { + const children = Object.keys(this._children) + const links = [] + + for (let i = 0; i < children.length; i++) { + let child = this._children[children[i]] + + if (typeof child.flush === 'function') { + for await (const entry of child.flush(child.path, ipld)) { + child = entry + + yield child + } + } + + links.push(new DAGLink(children[i], child.node.length || child.node.size, child.cid)) + } + + const unixfs = new UnixFS('directory') + let node = DAGNode.create(unixfs.marshal(), links) + const cid = await persist(node, ipld, this.options) + + this.cid = cid + this.size = node.size + + yield { + cid, + unixfs, + path, + node + } + } +} + +module.exports = DirFlat diff --git a/src/dir-sharded.js b/src/dir-sharded.js new file mode 100644 index 0000000..da7aeba --- /dev/null +++ b/src/dir-sharded.js @@ -0,0 +1,152 @@ +'use strict' + +const leftPad = require('left-pad') +const { + DAGLink, + DAGNode +} = require('ipld-dag-pb') +const UnixFS = require('ipfs-unixfs') +const multihashing = require('multihashing-async') +const Dir = require('./dir') +const persist = require('./utils/persist') +const Bucket = require('hamt-sharding') +const extend = require('deep-extend') + +const hashFn = async function (value) { + const hash = await multihashing(Buffer.from(value, 'utf8'), 'murmur3-128') + + // Multihashing inserts preamble of 2 bytes. Remove it. + // Also, murmur3 outputs 128 bit but, accidently, IPFS Go's + // implementation only uses the first 64, so we must do the same + // for parity.. + const justHash = hash.slice(2, 10) + const length = justHash.length + const result = Buffer.alloc(length) + // TODO: invert buffer because that's how Go impl does it + for (let i = 0; i < length; i++) { + result[length - i - 1] = justHash[i] + } + + return result +} +hashFn.code = 0x22 // TODO: get this from multihashing-async? + +const defaultOptions = { + hashFn: hashFn +} + +class DirSharded extends Dir { + constructor (props, options) { + options = extend({}, defaultOptions, options) + + super(props, options) + + this._bucket = Bucket(options) + } + + async put (name, value) { + await this._bucket.put(name, value) + } + + async get (name) { + return this._bucket.get(name) + } + + childCount () { + return this._bucket.leafCount() + } + + directChildrenCount () { + return this._bucket.childrenCount() + } + + onlyChild () { + return this._bucket.onlyChild() + } + + async * eachChildSeries () { + for await (const { key, value } of this._bucket.eachLeafSeries()) { + yield { + key, + child: value + } + } + } + + async * flush (path, ipld) { + for await (const entry of flush(path, this._bucket, ipld, this.options)) { + yield entry + } + } +} + +module.exports = (props, options) => { + return new DirSharded(props, options) +} + +module.exports.hashFn = hashFn + +async function * flush (path, bucket, ipld, options) { + const children = bucket._children + const links = [] + + for (let i = 0; i < children.length; i++) { + const child = children.get(i) + + if (!child) { + continue + } + + const labelPrefix = leftPad(i.toString(16).toUpperCase(), 2, '0') + + if (Bucket.isBucket(child)) { + let shard + + for await (const subShard of await flush('', child, ipld, options)) { + shard = subShard + } + + links.push(await new DAGLink(labelPrefix, shard.node.size, shard.cid)) + } else if (typeof child.value.flush === 'function') { + const dir = child.value + let flushedDir + + for await (const entry of dir.flush(dir.path, ipld)) { + flushedDir = entry + + yield flushedDir + } + + const label = labelPrefix + child.key + links.push(new DAGLink(label, flushedDir.node.size, flushedDir.cid)) + } else { + const value = child.value + + if (!value.node) { + continue + } + + const label = labelPrefix + child.key + const size = value.node.length || value.node.size || value.node.Size + + links.push(await new DAGLink(label, size, value.cid)) + } + } + + // go-ipfs uses little endian, that's why we have to + // reverse the bit field before storing it + const data = Buffer.from(children.bitField().reverse()) + const dir = new UnixFS('hamt-sharded-directory', data) + dir.fanout = bucket.tableSize() + dir.hashType = options.hashFn.code + + const node = DAGNode.create(dir.marshal(), links) + const cid = await persist(node, ipld, options) + + yield { + cid, + node, + unixfs: dir, + path + } +} diff --git a/src/importer/dir.js b/src/dir.js similarity index 53% rename from src/importer/dir.js rename to src/dir.js index fda1f7b..24a1023 100644 --- a/src/importer/dir.js +++ b/src/dir.js @@ -1,8 +1,8 @@ 'use strict' module.exports = class Dir { - constructor (props, _options) { - this._options = _options || {} + constructor (props, options) { + this.options = options || {} Object.assign(this, props) } } diff --git a/src/flat-to-shard.js b/src/flat-to-shard.js new file mode 100644 index 0000000..8bd4879 --- /dev/null +++ b/src/flat-to-shard.js @@ -0,0 +1,47 @@ +'use strict' + +const DirSharded = require('./dir-sharded') + +module.exports = async function flatToShard (child, dir, threshold, options) { + let newDir = dir + + if (dir.flat && dir.directChildrenCount() >= threshold) { + newDir = await convertToShard(dir, options) + } + + const parent = newDir.parent + + if (parent) { + if (newDir !== dir) { + if (child) { + child.parent = newDir + } + + await parent.put(newDir.parentKey, newDir) + } + + if (parent) { + return flatToShard(newDir, parent, threshold, options) + } + } + + return newDir +} + +async function convertToShard (oldDir, options) { + const newDir = DirSharded({ + root: oldDir.root, + dir: true, + parent: oldDir.parent, + parentKey: oldDir.parentKey, + path: oldDir.path, + dirty: oldDir.dirty, + flat: false + }, options) + + for await (const { key, child } of oldDir.eachChildSeries()) { + await newDir.put(key, child) + } + + return newDir +} diff --git a/src/importer/dir-flat.js b/src/importer/dir-flat.js deleted file mode 100644 index 22efc37..0000000 --- a/src/importer/dir-flat.js +++ /dev/null @@ -1,84 +0,0 @@ -'use strict' - -const asyncEachSeries = require('async/eachSeries') -const waterfall = require('async/waterfall') -const dagPB = require('ipld-dag-pb') -const UnixFS = require('ipfs-unixfs') -const DAGLink = dagPB.DAGLink -const DAGNode = dagPB.DAGNode -const Dir = require('./dir') -const persist = require('../utils/persist') - -class DirFlat extends Dir { - constructor (props, _options) { - super(props, _options) - this._children = {} - } - - put (name, value, callback) { - this.cid = undefined - this.size = undefined - this._children[name] = value - process.nextTick(callback) - } - - get (name, callback) { - process.nextTick(() => callback(null, this._children[name])) - } - - childCount () { - return Object.keys(this._children).length - } - - directChildrenCount () { - return this.childCount() - } - - onlyChild (callback) { - process.nextTick(() => callback(null, this._children[Object.keys(this._children)[0]])) - } - - eachChildSeries (iterator, callback) { - asyncEachSeries( - Object.keys(this._children), - (key, callback) => { - iterator(key, this._children[key], callback) - }, - callback - ) - } - - flush (path, ipld, source, callback) { - const links = Object.keys(this._children) - .map((key) => { - const child = this._children[key] - return new DAGLink(key, child.size, child.cid) - }) - - const dir = new UnixFS('directory') - - waterfall( - [ - (callback) => DAGNode.create(dir.marshal(), links, callback), - (node, callback) => persist(node, ipld, this._options, callback), - ({ cid, node }, callback) => { - this.cid = cid - this.size = node.size - const pushable = { - path: path, - size: node.size, - cid: cid - } - source.push(pushable) - callback(null, node) - } - ], - callback) - } -} - -module.exports = createDirFlat - -function createDirFlat (props, _options) { - return new DirFlat(props, _options) -} diff --git a/src/importer/dir-sharded.js b/src/importer/dir-sharded.js deleted file mode 100644 index 32a7936..0000000 --- a/src/importer/dir-sharded.js +++ /dev/null @@ -1,192 +0,0 @@ -'use strict' - -const leftPad = require('left-pad') -const whilst = require('async/whilst') -const waterfall = require('async/waterfall') -const dagPB = require('ipld-dag-pb') -const UnixFS = require('ipfs-unixfs') -const DAGLink = dagPB.DAGLink -const DAGNode = dagPB.DAGNode -const multihashing = require('multihashing-async') -const Dir = require('./dir') -const persist = require('../utils/persist') -const toPull = require('async-iterator-to-pull-stream') -const pull = require('pull-stream/pull') -const onEnd = require('pull-stream/sinks/on-end') -const asyncMap = require('pull-stream/throughs/async-map') -const Bucket = require('hamt-sharding') - -const hashFn = function (value) { - return new Promise((resolve, reject) => { - multihashing(value, 'murmur3-128', (err, hash) => { - if (err) { - reject(err) - } else { - // Multihashing inserts preamble of 2 bytes. Remove it. - // Also, murmur3 outputs 128 bit but, accidently, IPFS Go's - // implementation only uses the first 64, so we must do the same - // for parity.. - const justHash = hash.slice(2, 10) - const length = justHash.length - const result = Buffer.alloc(length) - // TODO: invert buffer because that's how Go impl does it - for (let i = 0; i < length; i++) { - result[length - i - 1] = justHash[i] - } - resolve(result) - } - }) - }) -} -hashFn.code = 0x22 // TODO: get this from multihashing-async? - -const defaultOptions = { - hashFn: hashFn -} - -class DirSharded extends Dir { - constructor (props, _options) { - const options = Object.assign({}, defaultOptions, _options) - super(props, options) - this._bucket = Bucket(options) - } - - async put (name, value, callback) { - if (!callback) { - console.info('wut') - } - - try { - await this._bucket.put(name, value) - - return callback() - } catch (err) { - return callback(err) - } - } - - async get (name, callback) { - try { - return callback(null, await this._bucket.get(name)) - } catch (err) { - return callback(err) - } - } - - childCount () { - return this._bucket.leafCount() - } - - directChildrenCount () { - return this._bucket.childrenCount() - } - - onlyChild (callback) { - try { - return callback(null, this._bucket.onlyChild()) - } catch (err) { - return callback(err) - } - } - - eachChildSeries (iterator, callback) { - pull( - toPull(this._bucket.eachLeafSeries()), - asyncMap((child, cb) => { - iterator(child.key, child.value, cb) - }), - onEnd(callback) - ) - } - - flush (path, ipld, source, callback) { - flush(this._options, this._bucket, path, ipld, source, (err, results) => { - if (err) { - return callback(err) - } else { - this.cid = results.cid - this.size = results.node.size - } - - callback(null, results) - }) - } -} - -module.exports = createDirSharded -module.exports.hashFn = hashFn - -function createDirSharded (props, _options) { - return new DirSharded(props, _options) -} - -function flush (options, bucket, path, ipld, source, callback) { - const children = bucket._children // TODO: intromission - let index = 0 - const links = [] - whilst( - () => index < children.length, - (callback) => { - const child = children.get(index) - if (child) { - collectChild(child, index, (err) => { - index++ - callback(err) - }) - } else { - index++ - callback() - } - }, - (err) => { - if (err) { - callback(err) - return // early - } - haveLinks(links, callback) - } - ) - - function collectChild (child, index, callback) { - const labelPrefix = leftPad(index.toString(16).toUpperCase(), 2, '0') - if (Bucket.isBucket(child)) { - flush(options, child, path, ipld, null, (err, { cid, node }) => { - if (err) { - callback(err) - return // early - } - links.push(new DAGLink(labelPrefix, node.size, cid)) - callback() - }) - } else { - const value = child.value - const label = labelPrefix + child.key - links.push(new DAGLink(label, value.size, value.cid)) - callback() - } - } - - function haveLinks (links, callback) { - // go-ipfs uses little endian, that's why we have to - // reverse the bit field before storing it - const data = Buffer.from(children.bitField().reverse()) - const dir = new UnixFS('hamt-sharded-directory', data) - dir.fanout = bucket.tableSize() - dir.hashType = options.hashFn.code - waterfall([ - (cb) => DAGNode.create(dir.marshal(), links, cb), - (node, cb) => persist(node, ipld, options, cb), - ({ cid, node }, cb) => { - const pushable = { - path: path, - size: node.size, - cid: cid - } - if (source) { - source.push(pushable) - } - cb(null, { cid, node }) - } - ], callback) - } -} diff --git a/src/importer/flat-to-shard.js b/src/importer/flat-to-shard.js deleted file mode 100644 index 3bae8b4..0000000 --- a/src/importer/flat-to-shard.js +++ /dev/null @@ -1,74 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') -const DirSharded = require('./dir-sharded') - -module.exports = flatToShard - -function flatToShard (child, dir, threshold, options, callback) { - maybeFlatToShardOne(dir, threshold, options, (err, newDir) => { - if (err) { - callback(err) - return // early - } - - const parent = newDir.parent - if (parent) { - waterfall([ - (callback) => { - if (newDir !== dir) { - if (child) { - child.parent = newDir - } - parent.put(newDir.parentKey, newDir, callback) - } else { - callback() - } - }, - (callback) => { - if (parent) { - flatToShard(newDir, parent, threshold, options, callback) - } else { - callback(null, newDir) - } - } - ], callback) - } else { - // no parent, we're done climbing tree - callback(null, newDir) - } - }) -} - -function maybeFlatToShardOne (dir, threshold, options, callback) { - if (dir.flat && dir.directChildrenCount() >= threshold) { - definitelyShardOne(dir, options, callback) - } else { - callback(null, dir) - } -} - -function definitelyShardOne (oldDir, options, callback) { - const newDir = DirSharded({ - root: oldDir.root, - dir: true, - parent: oldDir.parent, - parentKey: oldDir.parentKey, - path: oldDir.path, - dirty: oldDir.dirty, - flat: false - }, options) - - oldDir.eachChildSeries( - (key, value, callback) => { - newDir.put(key, value, callback) - }, - (err) => { - if (err) { - callback(err) - } else { - callback(err, newDir) - } - } - ) -} diff --git a/src/importer/index.js b/src/importer/index.js deleted file mode 100644 index 1bf0f0f..0000000 --- a/src/importer/index.js +++ /dev/null @@ -1,117 +0,0 @@ -'use strict' - -const pause = require('pull-pause') -const pull = require('pull-stream/pull') -const map = require('pull-stream/throughs/map') -const writable = require('pull-write') -const pushable = require('pull-pushable') -const assert = require('assert') -const setImmediate = require('async/setImmediate') -const DAGBuilder = require('../builder') -const createTreeBuilder = require('./tree-builder') -const chunkers = require('../chunker') - -const defaultOptions = { - chunker: 'fixed', - rawLeaves: false, - hashOnly: false, - cidVersion: 0, - hash: null, - leafType: 'file', - hashAlg: 'sha2-256' -} - -module.exports = function (ipld, _options) { - const options = Object.assign({}, defaultOptions, _options) - options.cidVersion = options.cidVersion || 0 - - if (options.cidVersion > 0 && _options.rawLeaves === undefined) { - // if the cid version is 1 or above, use raw leaves as this is - // what go does. - options.rawLeaves = true - } - - if (_options && _options.hash !== undefined && _options.rawLeaves === undefined) { - // if a non-default hash alg has been specified, use raw leaves as this is - // what go does. - options.rawLeaves = true - } - - const Chunker = chunkers[options.chunker] - assert(Chunker, 'Unknkown chunker named ' + options.chunker) - - let pending = 0 - const waitingPending = [] - - const entry = { - sink: writable( - (nodes, callback) => { - pending += nodes.length - nodes.forEach((node) => entry.source.push(node)) - setImmediate(callback) - }, - null, - 1, - (err) => entry.source.end(err) - ), - source: pushable() - } - - const dagStream = DAGBuilder(Chunker, ipld, options) - - const treeBuilder = createTreeBuilder(ipld, options) - const treeBuilderStream = treeBuilder.stream() - const pausable = pause(() => {}) - - // TODO: transform this entry -> pausable -> -> exit - // into a generic NPM package named something like pull-pause-and-drain - - pull( - entry, - pausable, - dagStream, - map((node) => { - pending-- - if (!pending) { - process.nextTick(() => { - while (waitingPending.length) { - waitingPending.shift()() - } - }) - } - return node - }), - treeBuilderStream - ) - - return { - sink: entry.sink, - source: treeBuilderStream.source, - flush: flush - } - - function flush (callback) { - pausable.pause() - - // wait until all the files entered were - // transformed into DAG nodes - if (!pending) { - proceed() - } else { - waitingPending.push(proceed) - } - - function proceed () { - treeBuilder.flush((err, hash) => { - if (err) { - treeBuilderStream.source.end(err) - callback(err) - return - } - - pausable.resume() - callback(null, hash) - }) - } - } -} diff --git a/src/importer/tree-builder.js b/src/importer/tree-builder.js deleted file mode 100644 index fc47844..0000000 --- a/src/importer/tree-builder.js +++ /dev/null @@ -1,215 +0,0 @@ -'use strict' - -const eachSeries = require('async/eachSeries') -const eachOfSeries = require('async/eachOfSeries') -const waterfall = require('async/waterfall') -const createQueue = require('async/queue') -const writable = require('pull-write') -const pushable = require('pull-pushable') -const DirFlat = require('./dir-flat') -const flatToShard = require('./flat-to-shard') -const Dir = require('./dir') -const toPathComponents = require('../utils/to-path-components') -const errCode = require('err-code') - -module.exports = createTreeBuilder - -const defaultOptions = { - wrap: false, - shardSplitThreshold: 1000, - onlyHash: false -} - -function createTreeBuilder (ipld, _options) { - const options = Object.assign({}, defaultOptions, _options) - - const queue = createQueue(consumeQueue, 1) - // returned stream - let stream = createStream() - - // root node - let tree = DirFlat({ - path: '', - root: true, - dir: true, - dirty: false, - flat: true - }, options) - - return { - flush: flushRoot, - stream: getStream - } - - function consumeQueue (action, callback) { - const args = action.args.concat(function () { - action.cb.apply(null, arguments) - callback() - }) - action.fn.apply(null, args) - } - - function getStream () { - return stream - } - - function createStream () { - const sink = writable(write, null, 1, ended) - const source = pushable() - - return { - sink: sink, - source: source - } - - function write (elems, callback) { - eachSeries( - elems, - (elem, callback) => { - queue.push({ - fn: addToTree, - args: [elem], - cb: (err) => { - if (err) { - callback(err) - } else { - source.push(elem) - callback() - } - } - }) - }, - callback - ) - } - - function ended (err) { - flushRoot((flushErr) => { - source.end(flushErr || err) - }) - } - } - - // ---- Add to tree - - function addToTree (elem, callback) { - const pathElems = toPathComponents(elem.path || '') - let parent = tree - const lastIndex = pathElems.length - 1 - - let currentPath = '' - eachOfSeries(pathElems, (pathElem, index, callback) => { - if (currentPath) { - currentPath += '/' - } - currentPath += pathElem - const last = (index === lastIndex) - parent.dirty = true - parent.cid = null - parent.size = null - - if (last) { - waterfall([ - (callback) => parent.put(pathElem, elem, callback), - (callback) => flatToShard(null, parent, options.shardSplitThreshold, options, callback), - (newRoot, callback) => { - tree = newRoot - callback() - } - ], callback) - } else { - parent.get(pathElem, (err, treeNode) => { - if (err) { - callback(err) - return // early - } - let dir = treeNode - if (!dir || !(dir instanceof Dir)) { - dir = DirFlat({ - dir: true, - parent: parent, - parentKey: pathElem, - path: currentPath, - dirty: true, - flat: true - }, options) - } - const parentDir = parent - parent = dir - parentDir.put(pathElem, dir, callback) - }) - } - }, callback) - } - - // ---- Flush - - function flushRoot (callback) { - queue.push({ - fn: flush, - args: ['', tree], - cb: (err, node) => { - if (err) { - callback(err) - } else { - callback(null, node && node.cid) - } - } - }) - } - - function flush (path, tree, callback) { - if (tree.dir) { - if (tree.root && tree.childCount() > 1 && !options.wrap) { - callback(errCode(new Error('detected more than one root'), 'EMORETHANONEROOT')) - return // early - } - tree.eachChildSeries( - (key, child, callback) => { - flush(path ? (path + '/' + key) : key, child, callback) - }, - (err) => { - if (err) { - callback(err) - return // early - } - flushDir(path, tree, callback) - }) - } else { - // leaf node, nothing to do here - process.nextTick(callback) - } - } - - function flushDir (path, tree, callback) { - // don't create a wrapping node unless the user explicitely said so - if (tree.root && !options.wrap) { - tree.onlyChild((err, onlyChild) => { - if (err) { - callback(err) - return // early - } - - callback(null, onlyChild) - }) - - return // early - } - - if (!tree.dirty) { - callback(null, tree.cid) - return // early - } - - // don't flush directory unless it's been modified - - tree.dirty = false - tree.flush(path, ipld, stream.source, (err, node) => { - if (err) { - callback(err) - } else { - callback(null, node) - } - }) - } -} diff --git a/src/index.js b/src/index.js index 5963cf3..cd23b2f 100644 --- a/src/index.js +++ b/src/index.js @@ -1,26 +1,101 @@ 'use strict' -const pull = require('pull-stream/pull') -const map = require('pull-stream/throughs/map') -const toPull = require('async-iterator-to-pull-stream') -const toIterator = require('pull-stream-to-async-iterator') -const importer = require('./importer') - -module.exports = function (source, ipld, options = {}) { - return toIterator( - pull( - toPull.source(source), - map(({ path, content }) => { - if (content && content[Symbol.asyncIterator]) { - content = toPull(content) - } - - return { - path, - content - } - }), - importer(ipld, options) - ) - ) +const { superstruct } = require('superstruct') +const dagBuilder = require('./dag-builder') +const treeBuilder = require('./tree-builder') +const mh = require('multihashes') + +const struct = superstruct({ + types: { + codec: v => ['dag-pb', 'dag-cbor', 'raw'].includes(v), + hashAlg: v => Object.keys(mh.names).includes(v), + leafType: v => ['file', 'raw'].includes(v) + } +}) + +const ChunkerOptions = struct({ + minChunkSize: 'number?', + maxChunkSize: 'number?', + avgChunkSize: 'number?', + window: 'number?', + polynomial: 'string?' +}, { + maxChunkSize: 262144, + avgChunkSize: 262144, + window: 16, + polynomial: '0x3DF305DFB2A805' +}) + +const BuilderOptions = struct({ + maxChildrenPerNode: 'number?', + layerRepeat: 'number?' +}, { + maxChildrenPerNode: 174, + layerRepeat: 4 +}) + +const Options = struct({ + chunker: struct.enum(['fixed', 'rabin']), + rawLeaves: 'boolean?', + hashOnly: 'boolean?', + strategy: struct.enum(['balanced', 'flat', 'trickle']), + reduceSingleLeafToSelf: 'boolean?', + codec: 'codec?', + hashAlg: 'hashAlg?', + leafType: 'leafType?', + cidVersion: 'number?', + progress: 'function?', + wrapWithDirectory: 'boolean?', + shardSplitThreshold: 'number?', + onlyHash: 'boolean?', + chunkerOptions: ChunkerOptions, + builderOptions: BuilderOptions, + + wrap: 'boolean?', + pin: 'boolean?', + recursive: 'boolean?', + ignore: 'array?', + hidden: 'boolean?', + preload: 'boolean?' +}, { + chunker: 'fixed', + strategy: 'balanced', + rawLeaves: false, + reduceSingleLeafToSelf: true, + codec: 'dag-pb', + hashAlg: 'sha2-256', + leafType: 'file', + cidVersion: 0, + progress: () => () => {}, + shardSplitThreshold: 1000 +}) + +module.exports = async function * (source, ipld, options = {}) { + const opts = Options(options) + + if (options.cidVersion > 0 && options.rawLeaves === undefined) { + // if the cid version is 1 or above, use raw leaves as this is + // what go does. + opts.rawLeaves = true + } + + if (options.hashAlg !== undefined && options.rawLeaves === undefined) { + // if a non-default hash alg has been specified, use raw leaves as this is + // what go does. + opts.rawLeaves = true + } + + // go-ifps trickle dag defaults to unixfs raw leaves, balanced dag defaults to file leaves + if (options.strategy === 'trickle' && !options.leafType) { + opts.leafType = 'raw' + } + + for await (const entry of treeBuilder(dagBuilder(source, ipld, opts), ipld, opts)) { + yield { + cid: entry.cid, + path: entry.path, + unixfs: entry.unixfs, + size: entry.size + } + } } diff --git a/src/tree-builder.js b/src/tree-builder.js new file mode 100644 index 0000000..7e19a65 --- /dev/null +++ b/src/tree-builder.js @@ -0,0 +1,92 @@ +'use strict' + +const DirFlat = require('./dir-flat') +const flatToShard = require('./flat-to-shard') +const Dir = require('./dir') +const toPathComponents = require('./utils/to-path-components') +const errCode = require('err-code') +const first = require('async-iterator-first') + +async function addToTree (elem, tree, options) { + const pathElems = toPathComponents(elem.path || '') + const lastIndex = pathElems.length - 1 + let parent = tree + let currentPath = '' + + for (let i = 0; i < pathElems.length; i++) { + const pathElem = pathElems[i] + + currentPath += `${currentPath ? '/' : ''}${pathElem}` + + const last = (i === lastIndex) + parent.dirty = true + parent.cid = null + parent.size = null + + if (last) { + await parent.put(pathElem, elem) + tree = await flatToShard(null, parent, options.shardSplitThreshold, options) + } else { + let dir = await parent.get(pathElem) + + if (!dir || !(dir instanceof Dir)) { + dir = new DirFlat({ + dir: true, + parent: parent, + parentKey: pathElem, + path: currentPath, + dirty: true, + flat: true + }, options) + } + + await parent.put(pathElem, dir) + + parent = dir + } + } + + return tree +} + +async function * treeBuilder (source, ipld, options) { + let tree = new DirFlat({ + root: true, + dir: true, + path: '', + dirty: true, + flat: true + }, options) + + for await (const entry of source) { + tree = await addToTree(entry, tree, options) + + yield entry + } + + if (tree) { + if (!options.wrapWithDirectory) { + if (tree.childCount() > 1) { + throw errCode(new Error('detected more than one root'), 'EMORETHANONEROOT') + } + + const unwrapped = await first(tree.eachChildSeries()) + + if (!unwrapped) { + return + } + + tree = unwrapped.child + } + + if (!tree.dir) { + return + } + + for await (const entry of tree.flush(tree.path, ipld)) { + yield entry + } + } +} + +module.exports = treeBuilder diff --git a/src/utils/persist.js b/src/utils/persist.js index 039361b..6393837 100644 --- a/src/utils/persist.js +++ b/src/utils/persist.js @@ -3,56 +3,23 @@ const mh = require('multihashes') const mc = require('multicodec') -const { - util: { - cid +const persist = async (node, ipld, options) => { + if (!options.codec && node.length) { + options.cidVersion = 1 + options.codec = 'raw' } -} = require('ipld-dag-pb') -const defaultOptions = { - cidVersion: 0, - hashAlg: 'sha2-256', - codec: 'dag-pb' -} - -const persist = (node, ipld, options, callback) => { - let cidVersion = options.cidVersion || defaultOptions.cidVersion - let hashAlg = options.hashAlg || defaultOptions.hashAlg - let codec = options.codec || defaultOptions.codec - - if (Buffer.isBuffer(node)) { - cidVersion = 1 - codec = 'raw' + if (isNaN(options.hashAlg)) { + options.hashAlg = mh.names[options.hashAlg] } - if (hashAlg !== 'sha2-256' && hashAlg !== mh.names['sha2-256']) { - cidVersion = 1 + if (options.hashAlg !== mh.names['sha2-256']) { + options.cidVersion = 1 } - if (isNaN(hashAlg)) { - hashAlg = mh.names[hashAlg] - } - - if (options.onlyHash) { - return cid(node, { - version: cidVersion, - hashAlg: hashAlg - }, (err, cid) => { - callback(err, { - cid, - node - }) - }) - } + const format = mc[options.codec.toUpperCase().replace(/-/g, '_')] - ipld.put(node, mc[codec.toUpperCase().replace(/-/g, '_')], { - cidVersion: cidVersion, - hashAlg: hashAlg - }) - .then((cid) => callback(null, { - cid, - node - }), callback) + return ipld.put(node, format, options) } module.exports = persist diff --git a/test/builder-balanced.spec.js b/test/builder-balanced.spec.js index 24c19c4..f064dd8 100644 --- a/test/builder-balanced.spec.js +++ b/test/builder-balanced.spec.js @@ -4,26 +4,14 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const toIterator = require('pull-stream-to-async-iterator') -const pullBuilder = require('../src/builder/balanced') +const builder = require('../src/dag-builder/file/balanced') const all = require('async-iterator-all') -const builder = (source, reduce, options) => { - return toIterator( - pull( - values(source), - pullBuilder(reduce, options) - ) - ) -} - -function reduce (leaves, callback) { +function reduce (leaves) { if (leaves.length > 1) { - callback(null, { children: leaves }) + return { children: leaves } } else { - callback(null, leaves[0]) + return leaves[0] } } diff --git a/test/builder-dir-sharding.spec.js b/test/builder-dir-sharding.spec.js index bd604f7..73dcd81 100644 --- a/test/builder-dir-sharding.spec.js +++ b/test/builder-dir-sharding.spec.js @@ -11,6 +11,7 @@ const IPLD = require('ipld') const inMemory = require('ipld-in-memory') const leftPad = require('left-pad') const all = require('async-iterator-all') +const last = require('async-iterator-last') describe('builder: directory sharding', () => { let ipld @@ -27,20 +28,25 @@ describe('builder: directory sharding', () => { describe('basic dirbuilder', () => { it('yields a non-sharded dir', async () => { + const content = Buffer.from('i have the best bytes') const nodes = await all(importer([{ path: 'a/b', - content: Buffer.from('i have the best bytes') + content }], ipld, { shardSplitThreshold: Infinity // never shard })) expect(nodes.length).to.equal(2) + expect(nodes[0].path).to.equal('a/b') expect(nodes[1].path).to.equal('a') - const node = await exporter(nodes[1].cid, ipld) + const dirNode = await exporter(nodes[1].cid, ipld) + expect(dirNode.unixfs.type).to.equal('directory') - expect(node.unixfs.type).to.equal('directory') + const fileNode = await exporter(nodes[0].cid, ipld) + expect(fileNode.unixfs.type).to.equal('file') + expect(Buffer.concat(await all(fileNode.content()))).to.deep.equal(content) }) it('yields a sharded dir', async () => { @@ -184,6 +190,7 @@ describe('builder: directory sharding', () => { while (pendingDepth && pending) { i++ const dir = [] + for (let d = 0; d < depth; d++) { dir.push('big') } @@ -204,11 +211,10 @@ describe('builder: directory sharding', () => { } } - const nodes = await all(importer(source, ipld)) - const last = nodes[nodes.length - 1] - expect(last.path).to.equal('big') + const node = await last(importer(source, ipld)) + expect(node.path).to.equal('big') - rootHash = last.cid + rootHash = node.cid }) it('imports a big dir', async () => { diff --git a/test/builder-flat.spec.js b/test/builder-flat.spec.js index b535f1f..1e3acc8 100644 --- a/test/builder-flat.spec.js +++ b/test/builder-flat.spec.js @@ -4,26 +4,14 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const toIterator = require('pull-stream-to-async-iterator') -const pullBuilder = require('../src/builder/flat') +const builder = require('../src/dag-builder/file/flat') const all = require('async-iterator-all') -const builder = (source, reduce, options) => { - return toIterator( - pull( - values(source), - pullBuilder(reduce, options) - ) - ) -} - -function reduce (leaves, callback) { +function reduce (leaves) { if (leaves.length > 1) { - callback(null, { children: leaves }) + return { children: leaves } } else { - callback(null, leaves[0]) + return leaves[0] } } diff --git a/test/builder-only-hash.spec.js b/test/builder-only-hash.spec.js index 33d0c58..2a08712 100644 --- a/test/builder-only-hash.spec.js +++ b/test/builder-only-hash.spec.js @@ -4,24 +4,11 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const createBuilder = require('../src/builder') -const FixedSizeChunker = require('../src/chunker/fixed-size') -const toIterator = require('pull-stream-to-async-iterator') +const builder = require('../src/dag-builder') const all = require('async-iterator-all') -const builder = (source, ipld, options) => { - return toIterator( - pull( - values(source), - createBuilder(FixedSizeChunker, ipld, options) - ) - ) -} - describe('builder: onlyHash', () => { let ipld @@ -36,14 +23,26 @@ describe('builder: onlyHash', () => { }) it('will only chunk and hash if passed an "onlyHash" option', async () => { - const nodes = await all(builder({ - path: '/foo.txt', + const nodes = await all(builder([{ + path: 'foo.txt', content: Buffer.from([0, 1, 2, 3, 4]) - }, ipld, { - onlyHash: true + }], ipld, { + onlyHash: true, + chunker: 'fixed', + strategy: 'balanced', + progress: () => {}, + leafType: 'file', + reduceSingleLeafToSelf: true, + wrap: true, + chunkerOptions: { + maxChunkSize: 1024 + }, + builderOptions: { + maxChildrenPerNode: 254 + } })) - expect(nodes.length).to.equal(2) + expect(nodes.length).to.equal(1) try { await ipld.get(nodes[0].cid) diff --git a/test/builder-trickle-dag.spec.js b/test/builder-trickle-dag.spec.js index ea07f43..3744acd 100644 --- a/test/builder-trickle-dag.spec.js +++ b/test/builder-trickle-dag.spec.js @@ -4,21 +4,9 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const trickleBuilder = require('../src/builder/trickle') -const toIterator = require('pull-stream-to-async-iterator') +const builder = require('../src/dag-builder/file/trickle') const all = require('async-iterator-all') -const builder = (source, options) => { - return toIterator( - pull( - values(source), - trickleBuilder(reduce, options) - ) - ) -} - const createValues = (max) => { const output = [] @@ -29,11 +17,11 @@ const createValues = (max) => { return output } -function reduce (leaves, callback) { +function reduce (leaves) { if (leaves.length > 1) { - setTimeout(() => callback(null, { children: leaves }), 10) + return { children: leaves } } else { - setTimeout(() => callback(null, leaves[0]), 10) + return leaves[0] } } @@ -44,15 +32,15 @@ const options = { describe('builder: trickle', () => { it('reduces one value into itself', async () => { - const result = await all(builder([1], options)) + const result = await all(builder([1], reduce, options)) - expect(result).to.be.eql([1]) + expect(result).to.deep.equal([1]) }) it('reduces 3 values into parent', async () => { - const result = await all(builder(createValues(3), options)) + const result = await all(builder(createValues(3), reduce, options)) - expect(result).to.be.eql([{ + expect(result).to.deep.equal([{ children: [ 0, 1, @@ -61,10 +49,10 @@ describe('builder: trickle', () => { }]) }) - it('reduces 6 values correclty', async () => { - const result = await all(builder(createValues(6), options)) + it('reduces 6 values correctly', async () => { + const result = await all(builder(createValues(6), reduce, options)) - expect(result).to.be.eql([{ + expect(result).to.deep.equal([{ children: [ 0, 1, @@ -81,9 +69,9 @@ describe('builder: trickle', () => { }) it('reduces 9 values correclty', async () => { - const result = await all(builder(createValues(9), options)) + const result = await all(builder(createValues(9), reduce, options)) - expect(result).to.be.eql([{ + expect(result).to.deep.equal([{ children: [ 0, 1, @@ -106,10 +94,10 @@ describe('builder: trickle', () => { }]) }) - it('reduces 12 values correclty', async () => { - const result = await all(builder(createValues(12), options)) + it('reduces 12 values correctly', async () => { + const result = await all(builder(createValues(12), reduce, options)) - expect(result).to.be.eql([{ + expect(result).to.deep.equal([{ children: [ 0, 1, @@ -139,10 +127,10 @@ describe('builder: trickle', () => { }]) }) - it('reduces 21 values correclty', async () => { - const result = await all(builder(createValues(21), options)) + it('reduces 21 values correctly', async () => { + const result = await all(builder(createValues(21), reduce, options)) - expect(result).to.be.eql([{ + expect(result).to.deep.equal([{ children: [ 0, 1, @@ -193,14 +181,15 @@ describe('builder: trickle', () => { }]) }) - it('forms correct trickle tree', async () => { - const result = await all(builder(createValues(100), options)) + it('reduces 68 values correclty', async () => { + const result = await all(builder(createValues(68), reduce, options)) - expect(result).to.be.eql([{ + expect(result).to.deep.equal([{ children: [ 0, 1, 2, + { children: [ 3, @@ -215,6 +204,7 @@ describe('builder: trickle', () => { 8 ] }, + { children: [ 9, @@ -257,6 +247,7 @@ describe('builder: trickle', () => { } ] }, + { children: [ 27, @@ -266,117 +257,208 @@ describe('builder: trickle', () => { children: [ 30, 31, - 32 + 32, + { + children: [ + 33, + 34, + 35 + ] + }, + { + children: [ + 36, + 37, + 38 + ] + } ] }, { children: [ - 33, - 34, - 35 + 39, + 40, + 41, + { + children: [ + 42, + 43, + 44 + ] + }, + { + children: [ + 45, + 46, + 47 + ] + } ] - }, + } + ] + }, + { + children: [ + 48, + 49, + 50, { children: [ - 36, - 37, - 38, + 51, + 52, + 53, { children: [ - 39, - 40, - 41 + 54, + 55, + 56 ] }, { children: [ - 42, - 43, - 44 + 57, + 58, + 59 ] } ] }, { children: [ - 45, - 46, - 47, + 60, + 61, + 62, { children: [ - 48, - 49, - 50 + 63, + 64, + 65 ] }, { children: [ - 51, - 52, - 53 + 66, + 67 ] } ] } ] + } + ] + }]) + }) + + it('reduces 93 values correclty', async () => { + const result = await all(builder(createValues(93), reduce, options)) + + expect(result).to.deep.equal([{ + children: [ + 0, + 1, + 2, + + { + children: [ + 3, + 4, + 5 + ] }, { children: [ - 54, - 55, - 56, + 6, + 7, + 8 + ] + }, + + { + children: [ + 9, + 10, + 11, { children: [ - 57, - 58, - 59 + 12, + 13, + 14 ] }, { children: [ - 60, - 61, - 62 + 15, + 16, + 17 + ] + } + ] + }, + { + children: [ + 18, + 19, + 20, + { + children: [ + 21, + 22, + 23 ] }, { children: [ - 63, - 64, - 65, + 24, + 25, + 26 + ] + } + ] + }, + + { + children: [ + 27, + 28, + 29, + { + children: [ + 30, + 31, + 32, { children: [ - 66, - 67, - 68 + 33, + 34, + 35 ] }, { children: [ - 69, - 70, - 71 + 36, + 37, + 38 ] } ] }, { children: [ - 72, - 73, - 74, + 39, + 40, + 41, { children: [ - 75, - 76, - 77 + 42, + 43, + 44 ] }, { children: [ - 78, - 79, - 80 + 45, + 46, + 47 ] } ] @@ -385,45 +467,108 @@ describe('builder: trickle', () => { }, { children: [ - 81, - 82, - 83, + 48, + 49, + 50, { children: [ - 84, - 85, - 86 + 51, + 52, + 53, + { + children: [ + 54, + 55, + 56 + ] + }, + { + children: [ + 57, + 58, + 59 + ] + } ] }, { children: [ - 87, - 88, - 89 + 60, + 61, + 62, + { + children: [ + 63, + 64, + 65 + ] + }, + { + children: [ + 66, + 67, + 68 + ] + } ] - }, + } + ] + }, + + { + children: [ + 69, + 70, + 71, { children: [ - 90, - 91, - 92, + 72, + 73, + 74, { children: [ - 93, - 94, - 95 + 75, + 76, + 77, + { + children: [ + 78, + 79, + 80 + ] + }, + { + children: [ + 81, + 82, + 83 + ] + } ] }, { children: [ - 96, - 97, - 98 + 84, + 85, + 86, + { + children: [ + 87, + 88, + 89 + ] + }, + { + children: [ + 90, + 91, + 92 + ] + } ] } ] - }, - 99 + } ] } ] diff --git a/test/builder.spec.js b/test/builder.spec.js index 1795664..8b9914f 100644 --- a/test/builder.spec.js +++ b/test/builder.spec.js @@ -4,26 +4,13 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') const mh = require('multihashes') const IPLD = require('ipld') const inMemory = require('ipld-in-memory') const UnixFS = require('ipfs-unixfs') -const createBuilder = require('../src/builder') -const FixedSizeChunker = require('../src/chunker/fixed-size') -const toIterator = require('pull-stream-to-async-iterator') +const builder = require('../src/dag-builder') const first = require('async-iterator-first') -const builder = (source, ipld, options) => { - return toIterator( - pull( - values(source), - createBuilder(FixedSizeChunker, ipld, options) - ) - ) -} - describe('builder', () => { let ipld @@ -38,18 +25,31 @@ describe('builder', () => { }) const testMultihashes = Object.keys(mh.names).slice(1, 40) + const opts = { + strategy: 'flat', + chunker: 'fixed', + leafType: 'file', + reduceSingleLeafToSelf: true, + progress: () => {}, + chunkerOptions: { + maxChunkSize: 262144 + } + } it('allows multihash hash algorithm to be specified', async () => { for (let i = 0; i < testMultihashes.length; i++) { const hashAlg = testMultihashes[i] - const options = { hashAlg, strategy: 'flat' } + const options = { + ...opts, + hashAlg + } const content = String(Math.random() + Date.now()) const inputFile = { path: content + '.txt', content: Buffer.from(content) } - const imported = await first(builder([Object.assign({}, inputFile)], ipld, options)) + const imported = await first(builder([inputFile], ipld, options)) expect(imported).to.exist() @@ -59,8 +59,8 @@ describe('builder', () => { // Fetch using hashAlg encoded multihash const node = await ipld.get(imported.cid) - const fetchedContent = UnixFS.unmarshal(node.data).data - expect(fetchedContent.equals(inputFile.content)).to.be.true() + const fetchedContent = UnixFS.unmarshal(node.Data).data + expect(fetchedContent).to.deep.equal(inputFile.content) } }) @@ -69,7 +69,10 @@ describe('builder', () => { for (let i = 0; i < testMultihashes.length; i++) { const hashAlg = testMultihashes[i] - const options = { hashAlg, strategy: 'flat' } + const options = { + ...opts, + hashAlg + } const content = String(Math.random() + Date.now()) const inputFile = { path: content + '.txt', @@ -88,7 +91,10 @@ describe('builder', () => { for (let i = 0; i < testMultihashes.length; i++) { const hashAlg = testMultihashes[i] - const options = { hashAlg, strategy: 'flat' } + const options = { + ...opts, + hashAlg + } const inputFile = { path: `${String(Math.random() + Date.now())}-dir`, content: null @@ -101,7 +107,7 @@ describe('builder', () => { // Fetch using hashAlg encoded multihash const node = await ipld.get(imported.cid) - const meta = UnixFS.unmarshal(node.data) + const meta = UnixFS.unmarshal(node.Data) expect(meta.type).to.equal('directory') } }) diff --git a/test/chunker-fixed-size.spec.js b/test/chunker-fixed-size.spec.js index ed3f0aa..d5d02a5 100644 --- a/test/chunker-fixed-size.spec.js +++ b/test/chunker-fixed-size.spec.js @@ -1,26 +1,14 @@ /* eslint-env mocha */ 'use strict' -const createChunker = require('../src/chunker/fixed-size') +const chunker = require('../src/chunker/fixed-size') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') const isNode = require('detect-node') -const toIterator = require('pull-stream-to-async-iterator') const all = require('async-iterator-all') const loadFixture = require('aegir/fixtures') -const rawFile = loadFixture('test/fixtures/1MiB.txt') - -const chunker = (source, chunkSize) => { - return toIterator( - pull( - values(source), - createChunker(chunkSize) - ) - ) -} +const rawFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/1MiB.txt') describe('chunker: fixed size', function () { this.timeout(30000) @@ -40,7 +28,9 @@ describe('chunker: fixed size', function () { b2.fill('b') b3.fill('c') - const chunks = await all(chunker([b1, b2, b3], 256)) + const chunks = await all(chunker([b1, b2, b3], { + maxChunkSize: 256 + })) expect(chunks).to.have.length(8) chunks.forEach((chunk) => { @@ -55,7 +45,9 @@ describe('chunker: fixed size', function () { for (let i = 0; i < (256 * 12); i++) { input.push(buf) } - const chunks = await all(chunker(input, 256)) + const chunks = await all(chunker(input, { + maxChunkSize: 256 + })) expect(chunks).to.have.length(12) chunks.forEach((chunk) => { @@ -65,7 +57,9 @@ describe('chunker: fixed size', function () { it('256 KiB chunks', async () => { const KiB256 = 262144 - const chunks = await all(chunker([rawFile], KiB256)) + const chunks = await all(chunker([rawFile], { + maxChunkSize: KiB256 + })) expect(chunks).to.have.length(4) chunks.forEach((chunk) => { @@ -77,7 +71,9 @@ describe('chunker: fixed size', function () { const KiB256 = 262144 let file = Buffer.concat([rawFile, Buffer.from('hello')]) - const chunks = await all(chunker([file], KiB256)) + const chunks = await all(chunker([file], { + maxChunkSize: KiB256 + })) expect(chunks).to.have.length(5) let counter = 0 diff --git a/test/chunker-rabin-browser.spec.js b/test/chunker-rabin-browser.spec.js index 85148be..a13eb8c 100644 --- a/test/chunker-rabin-browser.spec.js +++ b/test/chunker-rabin-browser.spec.js @@ -1,25 +1,13 @@ /* eslint-env mocha */ 'use strict' -const createChunker = require('../src/chunker/rabin') +const chunker = require('../src/chunker/rabin') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') const isNode = require('detect-node') -const toIterator = require('pull-stream-to-async-iterator') const all = require('async-iterator-all') -const chunker = (source, options) => { - return toIterator( - pull( - values(source), - createChunker(options) - ) - ) -} - describe('chunker: rabin browser', () => { before(function () { if (isNode) { @@ -28,22 +16,10 @@ describe('chunker: rabin browser', () => { }) it('returns an error', async () => { - const b1 = Buffer.alloc(2 * 256) - const b2 = Buffer.alloc(1 * 256) - const b3 = Buffer.alloc(5 * 256) - - b1.fill('a') - b2.fill('b') - b3.fill('c') - try { - await all(chunker([b1, b2, b3], { - minChunkSize: 48, - avgChunkSize: 96, - maxChunkSize: 192 - })) + await all(chunker()) } catch (err) { - expect(err.message).to.include('Rabin chunker not available') + expect(err.code).to.equal('EUNSUPPORTED') } }) }) diff --git a/test/chunker-rabin.spec.js b/test/chunker-rabin.spec.js index d33c9f4..be61304 100644 --- a/test/chunker-rabin.spec.js +++ b/test/chunker-rabin.spec.js @@ -1,28 +1,16 @@ /* eslint-env mocha */ 'use strict' -const createChunker = require('../src/chunker/rabin') +const chunker = require('../src/chunker/rabin') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') const loadFixture = require('aegir/fixtures') const os = require('os') const isNode = require('detect-node') -const toIterator = require('pull-stream-to-async-iterator') const all = require('async-iterator-all') -const rawFile = loadFixture('test/fixtures/1MiB.txt') - -const chunker = (source, options) => { - return toIterator( - pull( - values(source), - createChunker(options) - ) - ) -} +const rawFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/1MiB.txt') describe('chunker: rabin', function () { this.timeout(30000) @@ -49,7 +37,9 @@ describe('chunker: rabin', function () { const chunks = await all(chunker([b1, b2, b3], { minChunkSize: 48, avgChunkSize: 96, - maxChunkSize: 192 + maxChunkSize: 192, + window: 16, + polynomial: '0x3DF305DFB2A805' })) chunks.forEach((chunk) => { @@ -63,7 +53,11 @@ describe('chunker: rabin', function () { b1.fill('a') const chunks = await all(chunker([b1], { - avgChunkSize: 256 + maxChunkSize: 262144, + minChunkSize: 1, + avgChunkSize: 256, + window: 16, + polynomial: '0x3DF305DFB2A805' })) chunks.forEach((chunk) => { @@ -78,7 +72,9 @@ describe('chunker: rabin', function () { const opts = { minChunkSize: KiB256 / 3, avgChunkSize: KiB256, - maxChunkSize: KiB256 + (KiB256 / 2) + maxChunkSize: KiB256 + (KiB256 / 2), + window: 16, + polynomial: '0x3DF305DFB2A805' } const chunks = await all(chunker([file], opts)) diff --git a/test/hash-parity-with-go-ipfs.spec.js b/test/hash-parity-with-go-ipfs.spec.js index 48d5539..13d65ca 100644 --- a/test/hash-parity-with-go-ipfs.spec.js +++ b/test/hash-parity-with-go-ipfs.spec.js @@ -18,9 +18,9 @@ const strategies = [ ] const expectedHashes = { - flat: 'QmRgXEDv6DL8uchf7h9j8hAGG8Fq5r1UZ6Jy3TQAPxEb76', - balanced: 'QmVY1TFpjYKSo8LRG9oYgH4iy9AduwDvBGNhqap1Gkxme3', - trickle: 'QmYPsm9oVGjWECkT7KikZmrf8imggqKe8uS8Jco3qfWUCH' + flat: 'QmeJ9FRWKnXZQiX5CM1E8j4gpGbg6otpgajThqsbnBpoyD', + balanced: 'QmRdPboiJQoZ5cdazR9a8vGqdJvWg6M5bfdtUSKNHpuscj', + trickle: 'QmdZcefqMZ3tzdS4CRBN5s1c67eS3nQzN8TNXFBYfgofoy' } strategies.forEach(strategy => { diff --git a/test/helpers/collect-leaf-cids.js b/test/helpers/collect-leaf-cids.js index 087a7ed..4ef6a4e 100644 --- a/test/helpers/collect-leaf-cids.js +++ b/test/helpers/collect-leaf-cids.js @@ -4,7 +4,7 @@ module.exports = function (cid, ipld) { async function * traverse (cid) { const node = await ipld.get(cid) - if (Buffer.isBuffer(node) || !node.links.length) { + if (Buffer.isBuffer(node) || !node.Links.length) { yield { node, cid @@ -13,7 +13,7 @@ module.exports = function (cid, ipld) { return } - node.links.forEach(link => traverse(link.cid)) + node.Links.forEach(link => traverse(link.Hash)) } return traverse(cid) diff --git a/test/helpers/finite-pseudorandom-byte-stream.js b/test/helpers/finite-pseudorandom-byte-stream.js index 2a2f3e0..e8461b8 100644 --- a/test/helpers/finite-pseudorandom-byte-stream.js +++ b/test/helpers/finite-pseudorandom-byte-stream.js @@ -1,35 +1,24 @@ 'use strict' -const pull = require('pull-stream/pull') -const take = require('pull-stream/throughs/take') -const collect = require('pull-stream/sinks/collect') -const generate = require('pull-generate') - -const randomByteStream = require('./random-byte-stream') -const chunker = require('../../src/chunker/fixed-size') - const REPEATABLE_CHUNK_SIZE = 300000 -module.exports = function (maxSize, seed) { +module.exports = async function * (maxSize, seed) { const chunks = Math.ceil(maxSize / REPEATABLE_CHUNK_SIZE) - return pull( - generate(0, generator), - take(chunks) - ) + let emitted = 0 + const buf = Buffer.alloc(REPEATABLE_CHUNK_SIZE) - function generator (iteration, cb) { - if (iteration === 0) { - pull( - randomByteStream(seed), - chunker(REPEATABLE_CHUNK_SIZE), - take(1), - collect((err, results) => { - const result = results[0] - cb(err, result, result) - }) - ) - } else { - cb(null, iteration, iteration) + while (emitted !== chunks) { + for (let i = 0; i < buf.length; i++) { + buf[i] = 256 & Math.floor(random(seed) * 256) } + + yield buf + + emitted++ } } + +function random (seed) { + const x = Math.sin(seed) * 10000 + return x - Math.floor(x) +} diff --git a/test/helpers/random-byte-stream.js b/test/helpers/random-byte-stream.js index 8dd4bce..d575870 100644 --- a/test/helpers/random-byte-stream.js +++ b/test/helpers/random-byte-stream.js @@ -1,15 +1,11 @@ 'use strict' -module.exports = function randomByteStream (_seed) { - let seed = _seed - return (end, cb) => { - if (end) { - cb(end) - } else { - const r = Math.floor(random(seed) * 256) - seed = r - cb(null, Buffer.from([r])) - } +module.exports = async function * randomByteStream (seed) { + while (true) { + const r = Math.floor(random(seed) * 256) + seed = r + + yield Buffer.from([r]) } } diff --git a/test/import-export-nested-dir.spec.js b/test/import-export-nested-dir.spec.js index 6efb003..5f86375 100644 --- a/test/import-export-nested-dir.spec.js +++ b/test/import-export-nested-dir.spec.js @@ -6,7 +6,6 @@ chai.use(require('dirty-chai')) const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const values = require('pull-stream/sources/values') const all = require('async-iterator-all') const importer = require('../src') const exporter = require('ipfs-unixfs-exporter') @@ -28,33 +27,47 @@ describe('import and export: directory', () => { it('imports', async function () { this.timeout(20 * 1000) - const source = [ - { path: 'a/b/c/d/e', content: values([Buffer.from('banana')]) }, - { path: 'a/b/c/d/f', content: values([Buffer.from('strawberry')]) }, - { path: 'a/b/g', content: values([Buffer.from('ice')]) }, - { path: 'a/b/h', content: values([Buffer.from('cream')]) } - ] + const source = [{ + path: 'a/b/c/d/e', + content: Buffer.from('banana') + }, { + path: 'a/b/c/d/f', + content: Buffer.from('strawberry') + }, { + path: 'a/b/g', + content: Buffer.from('ice') + }, { + path: 'a/b/h', + content: Buffer.from('cream') + }] const files = await all(importer(source, ipld)) - expect(files.map(normalizeNode).sort(byPath)).to.be.eql([ - { path: 'a/b/h', - multihash: 'QmWHMpCtdNjemT2F3SjyrmnBXQXwEohaZd4apcbFBhbFRC' }, - { path: 'a/b/g', - multihash: 'QmQGwYzzTPcbqTiy2Nbp88gqqBqCWY4QZGfen45LFZkD5n' }, - { path: 'a/b/c/d/f', - multihash: 'QmNVHs2dy7AjGUotsubWVncRsD3SpRXm8MgmCCQTVdVACz' }, - { path: 'a/b/c/d/e', - multihash: 'QmYPbDKwc7oneCcEc6BcRSN5GXthTGWUCd19bTCyP9u3vH' }, - { path: 'a/b/c/d', - multihash: 'QmQGDXr3ysARM38n7h79Tx7yD3YxuzcnZ1naG71WMojPoj' }, - { path: 'a/b/c', - multihash: 'QmYTVcjYpN3hQLtJstCPE8hhEacAYjWAuTmmAAXoonamuE' }, - { path: 'a/b', - multihash: 'QmWyWYxq1GD9fEyckf5LrJv8hMW35CwfWwzDBp8bTw3NQj' }, - { path: 'a', - multihash: rootHash } - ]) + expect(files.map(normalizeNode).sort(byPath)).to.be.eql([{ + path: 'a/b/h', + multihash: 'QmWHMpCtdNjemT2F3SjyrmnBXQXwEohaZd4apcbFBhbFRC' + }, { + path: 'a/b/g', + multihash: 'QmQGwYzzTPcbqTiy2Nbp88gqqBqCWY4QZGfen45LFZkD5n' + }, { + path: 'a/b/c/d/f', + multihash: 'QmNVHs2dy7AjGUotsubWVncRsD3SpRXm8MgmCCQTVdVACz' + }, { + path: 'a/b/c/d/e', + multihash: 'QmYPbDKwc7oneCcEc6BcRSN5GXthTGWUCd19bTCyP9u3vH' + }, { + path: 'a/b/c/d', + multihash: 'QmQGDXr3ysARM38n7h79Tx7yD3YxuzcnZ1naG71WMojPoj' + }, { + path: 'a/b/c', + multihash: 'QmYTVcjYpN3hQLtJstCPE8hhEacAYjWAuTmmAAXoonamuE' + }, { + path: 'a/b', + multihash: 'QmWyWYxq1GD9fEyckf5LrJv8hMW35CwfWwzDBp8bTw3NQj' + }, { + path: 'a', + multihash: rootHash + }]) }) it('exports', async function () { @@ -63,16 +76,19 @@ describe('import and export: directory', () => { const dir = await exporter(rootHash, ipld) const files = await recursiveExport(dir, rootHash) - expect(files.sort(byPath)).to.eql([ - { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/h', - content: 'cream' }, - { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/g', - content: 'ice' }, - { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/f', - content: 'strawberry' }, - { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/e', - content: 'banana' } - ]) + expect(files.sort(byPath)).to.eql([{ + path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/h', + content: 'cream' + }, { + path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/g', + content: 'ice' + }, { + path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/f', + content: 'strawberry' + }, { + path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/e', + content: 'banana' + }]) }) }) diff --git a/test/import-export.spec.js b/test/import-export.spec.js index b5763b7..86f4435 100644 --- a/test/import-export.spec.js +++ b/test/import-export.spec.js @@ -8,7 +8,8 @@ const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') const loadFixture = require('aegir/fixtures') -const bigFile = loadFixture('test/fixtures/1.2MiB.txt') +const isNode = require('detect-node') +const bigFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/1.2MiB.txt') const importer = require('../src') const exporter = require('ipfs-unixfs-exporter') @@ -39,7 +40,7 @@ describe('import and export', function () { }) it('imports and exports', async () => { - const path = strategy + '-big.dat' + const path = `${strategy}-big.dat` const values = [{ path: path, content: bigFile }] for await (const file of importer(values, ipld, importerOptions)) { diff --git a/test/importer-flush.spec.js b/test/importer-flush.spec.js deleted file mode 100644 index 19abeeb..0000000 --- a/test/importer-flush.spec.js +++ /dev/null @@ -1,204 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const createImporter = require('../src') - -const chai = require('chai') -chai.use(require('dirty-chai')) -const expect = chai.expect -const IPLD = require('ipld') -const inMemory = require('ipld-in-memory') -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const map = require('pull-stream/throughs/map') -const collect = require('pull-stream/sinks/collect') -const pushable = require('pull-pushable') - -describe.skip('importer: flush', () => { - let ipld - - before((done) => { - inMemory(IPLD, (err, resolver) => { - expect(err).to.not.exist() - - ipld = resolver - - done() - }) - }) - - it('can push a single root file and flush yields no dirs', (done) => { - const source = pushable() - const importer = createImporter(ipld) - pull( - source, - importer, - map(node => { - expect(node.path).to.be.eql('a') - return node - }), - collect((err, files) => { - expect(err).to.not.exist() - expect(files.length).to.be.eql(1) - done() - }) - ) - - source.push({ - path: 'a', - content: values([Buffer.from('hey')]) - }) - - importer.flush((err, hash) => { - expect(err).to.not.exist() - expect(Buffer.isBuffer(hash)).to.be.true() - source.end() - }) - }) - - it('can push a nested file and flush yields parent dir', (done) => { - const source = pushable() - const importer = createImporter(ipld) - let count = 0 - pull( - source, - importer, - map(function (node) { - count++ - if (count === 1) { - expect(node.path).to.be.eql('b/c') - } else if (count === 2) { - expect(node.path).to.be.eql('b') - } - return node - }), - collect((err, files) => { - expect(err).to.not.exist() - expect(count).to.be.eql(2) - done() - }) - ) - - source.push({ - path: 'b/c', - content: values([Buffer.from('hey')]) - }) - - importer.flush((err, hash) => { - expect(err).to.not.exist() - expect(Buffer.isBuffer(hash)).to.be.true() - source.end() - }) - }) - - it('can flush many times, always coherent', (done) => { - const maxDepth = 4 - const maxEntriesPerDir = 3 - - let count = 0 - const tree = { children: {}, path: '', depth: 0, yielded: true } - let currentDir = tree - - const source = pushable() - const importer = createImporter(ipld) - - pull( - source, - importer, - map((node) => { - count++ - markDirAsYielded(node) - return node - }), - collect((err, files) => { - expect(err).to.not.exist() - expect(count).to.be.eql(2) - done() - }) - ) - - pushAndFlush() - - function pushAndFlush () { - const childCount = Object.keys(currentDir.children).length - const newDirName = childCount.toString() - const dirPath = currentDir.path + (currentDir.depth > 0 ? '/' : '') + newDirName - const newDir = { - children: {}, - path: dirPath, - depth: currentDir.depth + 1, - yielded: false, - parent: currentDir - } - currentDir.children[newDirName] = newDir - markAncestorsAsDirty(currentDir) - - const filePath = dirPath + '/filename' - const file = { - path: filePath, - content: values([Buffer.from('file with path ' + filePath)]) - } - source.push(file) - if (currentDir.depth === 0 || childCount + 1 === maxEntriesPerDir) { - currentDir = newDir - } - importer.flush((err, hash) => { - expect(err).to.not.exist() - expect(Buffer.isBuffer(hash)).to.be.true() - testAllYielded(tree) - if (currentDir.depth < maxDepth) { - pushAndFlush() - } else { - expect(count).to.be.eql(38) - done() - } - }) - } - - function markDirAsYielded (node) { - const dir = findDir(tree, node.path) - if (node.path === dir.path) { - expect(dir.yielded).to.be.false() - dir.yielded = true - } - } - - function findDir (tree, path) { - const pathElems = path.split('/').filter(notEmpty) - const child = tree.children[pathElems.shift()] - if (!child) { - return tree - } - if (pathElems.length) { - return findDir(child, pathElems.join('/')) - } else { - return child - } - } - - function testAllYielded (tree) { - if (tree.depth) { - expect(tree.yielded).to.be.true() - } - const childrenNames = Object.keys(tree.children) - childrenNames.forEach((childName) => { - const child = tree.children[childName] - testAllYielded(child) - }) - } - - function markAncestorsAsDirty (dir) { - dir.yielded = false - while (dir) { - dir = dir.parent - if (dir) { - dir.yielded = false - } - } - } - }) -}) - -function notEmpty (str) { - return Boolean(str) -} diff --git a/test/importer.spec.js b/test/importer.spec.js index 08f2899..e23e651 100644 --- a/test/importer.spec.js +++ b/test/importer.spec.js @@ -14,30 +14,33 @@ const inMemory = require('ipld-in-memory') const UnixFs = require('ipfs-unixfs') const collectLeafCids = require('./helpers/collect-leaf-cids') const loadFixture = require('aegir/fixtures') -const bigFile = loadFixture('test/fixtures/1.2MiB.txt') -const smallFile = loadFixture('test/fixtures/200Bytes.txt') +const isNode = require('detect-node') +const bigFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/1.2MiB.txt') +const smallFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/200Bytes.txt') +const all = require('async-iterator-all') +const first = require('async-iterator-first') function stringifyMh (files) { return files.map((file) => { - file.cid = file.cid.toBaseEncodedString() - return file + return { + ...file, + cid: file.cid.toBaseEncodedString() + } }) } const baseFiles = { '200Bytes.txt': { - path: '200Bytes.txt', cid: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', - size: 211, - name: '', - leafSize: 200 + size: 200, + type: 'file', + path: '200Bytes.txt' }, '1.2MiB.txt': { - path: '1.2MiB.txt', - cid: 'QmbPN6CXXWpejfQgnRYnMQcVYkFHEntHWqLNQjbkatYCh1', - size: 1328062, - name: '', - leafSize: 1258000 + cid: 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q', + size: 1258000, + type: 'file', + path: '1.2MiB.txt' } } @@ -45,101 +48,111 @@ const strategyBaseFiles = { flat: baseFiles, balanced: extend({}, baseFiles, { '1.2MiB.txt': { - cid: 'QmeEGqUisUD2T6zU96PrZnCkHfXCGuQeGWKu4UoSuaZL3d', - size: 1335420 + cid: 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q', + type: 'file' } }), trickle: extend({}, baseFiles, { '1.2MiB.txt': { - cid: 'QmaiSohNUt1rBf2Lqz6ou54NHVPTbXbBoPuq9td4ekcBx4', - size: 1334599 + cid: 'QmfAxsHrpaLLuhbqqbo9KQyvQNawMnVSwutYoJed75pnco', + type: 'file' } }) } const strategies = [ 'flat', - 'balanced', - 'trickle' + 'balanced' + // 'trickle' ] const strategyOverrides = { balanced: { 'foo-big': { + cid: 'QmaFgyFJUP4fxFySJCddg2Pj6rpwSywopWk87VEVv52RSj', path: 'foo-big', - cid: 'QmQ1S6eEamaf4t948etp8QiYQ9avrKCogiJnPRgNkVreLv', - size: 1335478 + size: 1335478, + type: 'directory' }, pim: { - cid: 'QmUpzaN4Jio2GB3HoPSRCMQD5EagdMWjSEGD4SGZXaCw7W', - size: 1335744 + cid: 'QmY8a78tx6Tk6naDgWCgTsd9EqGrUJRrH7dDyQhjyrmH2i', + path: 'pim', + size: 1335744, + type: 'directory' }, 'pam/pum': { - cid: 'QmUpzaN4Jio2GB3HoPSRCMQD5EagdMWjSEGD4SGZXaCw7W', - size: 1335744 + cid: 'QmY8a78tx6Tk6naDgWCgTsd9EqGrUJRrH7dDyQhjyrmH2i', + path: 'pam/pum', + size: 1335744, + type: 'directory' }, pam: { - cid: 'QmVoVD4fEWFLJLjvRCg4bGrziFhgECiaezp79AUfhuLgno', - size: 2671269 + cid: 'QmRgdtzNx1H1BPJqShdhvWZ2D4DA2HUgZJ3XLtoXei27Av', + path: 'pam', + size: 2671269, + type: 'directory' } }, trickle: { 'foo-big': { - path: 'foo-big', cid: 'QmPh6KSS7ghTqzgWhaoCiLoHFPF7HGqUxx7q9vcM5HUN4U', - size: 1334657 + path: 'foo-big', + size: 1334657, + type: 'directory' }, pim: { cid: 'QmPAn3G2x2nrq4A1fu2XUpwWtpqG4D1YXFDrU615NHvJbr', - size: 1334923 + path: 'pim', + size: 1334923, + type: 'directory' }, 'pam/pum': { cid: 'QmPAn3G2x2nrq4A1fu2XUpwWtpqG4D1YXFDrU615NHvJbr', - size: 1334923 + path: 'pam/pum', + size: 1334923, + type: 'directory' }, pam: { cid: 'QmZTJah1xpG9X33ZsPtDEi1tYSHGDqQMRHsGV5xKzAR2j4', - size: 2669627 + path: 'pam', + size: 2669627, + type: 'directory' } } } -const checkLeafNodeTypes = async (ipld, options, expected, done) => { - const files = [] - - for await (const file of importer([{ - path: '/foo', +const checkLeafNodeTypes = async (ipld, options, expected) => { + const file = await first(importer([{ + path: 'foo', content: Buffer.alloc(262144 + 5).fill(1) - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) - const node = await ipld.get(files[0].cid) - const meta = UnixFs.unmarshal(node.data) + const node = await ipld.get(file.cid) + const meta = UnixFs.unmarshal(node.Data) expect(meta.type).to.equal('file') - expect(node.links.length).to.equal(2) + expect(node.Links.length).to.equal(2) const linkedNodes = await Promise.all( - node.links.map(link => ipld.get(link.cid)) + node.Links.map(link => ipld.get(link.Hash)) ) linkedNodes.forEach(node => { - const meta = UnixFs.unmarshal(node.data) + const meta = UnixFs.unmarshal(node.Data) expect(meta.type).to.equal(expected) }) } const checkNodeLinks = async (ipld, options, expected) => { for await (const file of importer([{ - path: '/foo', + path: 'foo', content: Buffer.alloc(100).fill(1) }], ipld, options)) { const node = await ipld.get(file.cid) - const meta = UnixFs.unmarshal(node.data) + const meta = UnixFs.unmarshal(node.Data) expect(meta.type).to.equal('file') - expect(node.links.length).to.equal(expected) + expect(node.Links.length).to.equal(expected) } } @@ -152,20 +165,23 @@ strategies.forEach((strategy) => { foo: { path: 'foo', cid: 'QmQrb6KKWGo8w7zKfx2JksptY6wN7B2ysSBdKZr4xMU36d', - size: 320 + size: 320, + type: 'directory' }, 'foo/bar': { path: 'foo/bar', cid: 'Qmf5BQbTUyUAvd6Ewct83GYGnE1F6btiC3acLhR8MDxgkD', - size: 270 + size: 270, + type: 'directory' }, 'foo-big/1.2MiB.txt': extend({}, baseFiles['1.2MiB.txt'], { path: 'foo-big/1.2MiB.txt' }), 'foo-big': { path: 'foo-big', - cid: 'Qma6JU3FoXU9eAzgomtmYPjzFBwVc2rRbECQpmHFiA98CJ', - size: 1328120 + cid: 'QmaFgyFJUP4fxFySJCddg2Pj6rpwSywopWk87VEVv52RSj', + size: 1328120, + type: 'directory' }, 'pim/200Bytes.txt': extend({}, baseFiles['200Bytes.txt'], { path: 'pim/200Bytes.txt' @@ -175,21 +191,27 @@ strategies.forEach((strategy) => { }), pim: { path: 'pim', - cid: 'QmNk8VPGb3fkAQgoxctXo4Wmnr4PayFTASy4MiVXTtXqiA', - size: 1328386 + cid: 'QmY8a78tx6Tk6naDgWCgTsd9EqGrUJRrH7dDyQhjyrmH2i', + size: 1328386, + type: 'directory' }, 'empty-dir': { path: 'empty-dir', cid: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn', - size: 4 + size: 4, + type: 'directory' }, 'pam/pum': { - cid: 'QmNk8VPGb3fkAQgoxctXo4Wmnr4PayFTASy4MiVXTtXqiA', - size: 1328386 + cid: 'QmY8a78tx6Tk6naDgWCgTsd9EqGrUJRrH7dDyQhjyrmH2i', + path: 'pam/pum', + size: 1328386, + type: 'directory' }, pam: { - cid: 'QmPAixYTaYnPe795fcWcuRpo6tfwHgRKNiBHpMzoomDVN6', - size: 2656553 + cid: 'QmRgdtzNx1H1BPJqShdhvWZ2D4DA2HUgZJ3XLtoXei27Av', + path: 'pam', + size: 2656553, + type: 'directory' }, '200Bytes.txt with raw leaves': extend({}, baseFiles['200Bytes.txt'], { cid: 'zb2rhXrz1gkCv8p4nUDZRohY6MzBE9C3HVTVDP72g6Du3SD9Q', @@ -199,16 +221,32 @@ strategies.forEach((strategy) => { const expected = extend({}, defaultResults, strategies[strategy]) + const expectFiles = (actualFiles, expectedFiles) => { + expect(actualFiles.length).to.equal(expectedFiles.length) + + for (let i = 0; i < expectedFiles.length; i++) { + const expectedFile = expected[expectedFiles[i]] + const actualFile = actualFiles[i] + + expect(actualFile.path).to.equal(expectedFile.path) + expect(actualFile.cid.toBaseEncodedString('base58btc')).to.equal(expectedFile.cid) + + if (actualFile.unixfs) { + expect(actualFile.unixfs.type).to.equal(expectedFile.type) + + if (actualFile.unixfs.type === 'file') { + expect(actualFile.unixfs.fileSize()).to.equal(expectedFile.size) + } + } + } + } + describe('importer: ' + strategy, function () { this.timeout(30 * 1000) let ipld const options = { - strategy: strategy, - maxChildrenPerNode: 10, - chunkerOptions: { - maxChunkSize: 1024 - } + strategy: strategy } before((done) => { @@ -221,56 +259,45 @@ strategies.forEach((strategy) => { }) }) - it('fails on bad input', async () => { + it('fails on bad content', async () => { try { - for await (const _ of importer([{ // eslint-disable-line no-unused-vars + await all(importer([{ path: '200Bytes.txt', - content: 'banana' - }], ipld, options)) { - // ...falala - } - + content: 7 + }], ipld, options)) throw new Error('No error was thrown') } catch (err) { expect(err.code).to.equal('EINVALIDCONTENT') } }) - it('survives bad progress option', async () => { - let file - - for await (const f of importer([{ - path: '200Bytes.txt', - content: Buffer.from([0, 1, 2]) - }], ipld, { - ...options, - progress: null - })) { - file = f + it('fails on an iterator that yields bad content', async () => { + try { + await all(importer([{ + path: '200Bytes.txt', + content: { + [Symbol.asyncIterator]: async function * () { + yield 7 + } + } + }], ipld, options)) + throw new Error('No error was thrown') + } catch (err) { + expect(err.code).to.equal('EINVALIDCONTENT') } - - expect(file).to.be.ok() }) it('doesn\'t yield anything on empty source', async () => { - const files = [] - - for await (const file of importer([], ipld, options)) { - files.push(file) - } + const files = await all(importer([], ipld, options)) expect(files).to.be.empty() }) it('doesn\'t yield anything on empty file', async () => { - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: 'emptyfile', content: Buffer.alloc(0) - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) expect(files.length).to.eql(1) @@ -280,15 +307,13 @@ strategies.forEach((strategy) => { it('fails on more than one root', async () => { try { - for await (const _ of importer([{ // eslint-disable-line no-unused-vars - path: '/beep/200Bytes.txt', + await all(importer([{ + path: 'beep/200Bytes.txt', content: smallFile }, { - path: '/boop/200Bytes.txt', + path: 'boop/200Bytes.txt', content: bigFile - }], ipld, options)) { - // ...falala - } + }], ipld, options)) throw new Error('No error was thrown') } catch (err) { @@ -296,16 +321,25 @@ strategies.forEach((strategy) => { } }) + it('accepts strings as content', async () => { + const content = 'I am a string' + const res = await all(importer([{ + path: '200Bytes.txt', + content + }], ipld, options)) + + const file = await exporter(res[0].cid, ipld) + const fileContent = await all(file.content()) + + expect(fileContent.toString()).to.equal(content) + }) + it('small file with an escaped slash in the title', async () => { const filePath = `small-\\/file-${Math.random()}.txt` - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: filePath, content: smallFile - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) expect(files.length).to.equal(1) expect(files[0].path).to.equal(filePath) @@ -313,146 +347,128 @@ strategies.forEach((strategy) => { it('small file with square brackets in the title', async () => { const filePath = `small-[v]-file-${Math.random()}.txt` - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: filePath, content: smallFile - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) expect(files.length).to.equal(1) expect(files[0].path).to.equal(filePath) }) - it('small file (smaller than a chunk)', async () => { - const files = [] - - for await (const file of importer([{ + it('small file as buffer (smaller than a chunk)', async () => { + const files = await all(importer([{ path: '200Bytes.txt', content: smallFile - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) - expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt']]) + expectFiles(files, [ + '200Bytes.txt' + ]) }) - it('small file (smaller than a chunk) with raw leaves', async () => { - const files = [] - - for await (const file of importer([{ + it('small file as array (smaller than a chunk)', async () => { + const files = await all(importer([{ path: '200Bytes.txt', - content: smallFile - }], ipld, { - ...options, - rawLeaves: true - })) { - files.push(file) - } + content: Array.from(smallFile) + }], ipld, options)) - expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt with raw leaves']]) + expectFiles(files, [ + '200Bytes.txt' + ]) }) - it('small file as buffer (smaller than a chunk)', async () => { - const files = [] + it('small file as string (smaller than a chunk)', async () => { + const file = await first(importer([{ + path: '200Bytes.txt', + content: 'this is a file\n' + }], ipld, options)) - for await (const file of importer([{ + expect(file.cid.toBaseEncodedString()).to.equal('QmZMb7HWpbevpcdhbUV1ZZgdji8vh5uQ13KxczChGrK9Rd') + }) + + it('small file (smaller than a chunk) with raw leaves', async () => { + const files = await all(importer([{ path: '200Bytes.txt', content: smallFile - }], ipld, options)) { - files.push(file) - } + }], ipld, { + ...options, + rawLeaves: true + })) - expect(stringifyMh(files)).to.be.eql([expected['200Bytes.txt']]) + expectFiles(files, [ + '200Bytes.txt with raw leaves' + ]) }) it('small file (smaller than a chunk) inside a dir', async () => { - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: 'foo/bar/200Bytes.txt', content: smallFile - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) - expect(files.length).to.equal(3) - stringifyMh(files).forEach((file) => { - expect(file).to.deep.equal(expected[file.path]) - }) + expectFiles(files, [ + 'foo/bar/200Bytes.txt', + 'foo/bar', + 'foo' + ]) }) it('file bigger than a single chunk', async () => { this.timeout(60 * 1000) - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: '1.2MiB.txt', content: bigFile - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) - expect(stringifyMh(files)).to.be.eql([expected['1.2MiB.txt']]) + expectFiles(files, [ + '1.2MiB.txt' + ]) }) it('file bigger than a single chunk inside a dir', async () => { this.timeout(60 * 1000) - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: 'foo-big/1.2MiB.txt', content: bigFile - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) - expect(stringifyMh(files)).to.deep.equal([ - expected['foo-big/1.2MiB.txt'], - expected['foo-big'] + expectFiles(files, [ + 'foo-big/1.2MiB.txt', + 'foo-big' ]) }) it('empty directory', async () => { - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: 'empty-dir' - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) - expect(stringifyMh(files)).to.be.eql([expected['empty-dir']]) + expectFiles(files, [ + 'empty-dir' + ]) }) it('directory with files', async () => { - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: 'pim/200Bytes.txt', content: smallFile }, { path: 'pim/1.2MiB.txt', content: bigFile - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) - expect(stringifyMh(files)).be.eql([ - expected['pim/200Bytes.txt'], - expected['pim/1.2MiB.txt'], - expected.pim] - ) + expectFiles(files, [ + 'pim/200Bytes.txt', + 'pim/1.2MiB.txt', + 'pim' + ]) }) it('nested directory (2 levels deep)', async () => { - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: 'pam/pum/200Bytes.txt', content: smallFile }, { @@ -461,50 +477,42 @@ strategies.forEach((strategy) => { }, { path: 'pam/1.2MiB.txt', content: bigFile - }], ipld, options)) { - files.push(file) - } + }], ipld, options)) + + const result = stringifyMh(files) + + expect(result.length).to.equal(5) - stringifyMh(files).forEach(eachFile) + result.forEach(eachFile) function eachFile (file) { if (file.path === 'pam/pum/200Bytes.txt') { - expect(file.cid).to.be.eql(expected['200Bytes.txt'].cid) - expect(file.size).to.be.eql(expected['200Bytes.txt'].size) - } - if (file.path === 'pam/pum/1.2MiB.txt') { - expect(file.cid).to.be.eql(expected['1.2MiB.txt'].cid) - expect(file.size).to.be.eql(expected['1.2MiB.txt'].size) - } - if (file.path === 'pam/pum') { - const dir = expected['pam/pum'] - expect(file.cid).to.be.eql(dir.cid) - expect(file.size).to.be.eql(dir.size) - } - if (file.path === 'pam/1.2MiB.txt') { - expect(file.cid).to.be.eql(expected['1.2MiB.txt'].cid) - expect(file.size).to.be.eql(expected['1.2MiB.txt'].size) - } - if (file.path === 'pam') { - const dir = expected.pam - expect(file.cid).to.be.eql(dir.cid) - expect(file.size).to.be.eql(dir.size) + expect(file.cid).to.equal(expected['200Bytes.txt'].cid) + expect(file.unixfs.fileSize()).to.equal(expected['200Bytes.txt'].size) + } else if (file.path === 'pam/pum/1.2MiB.txt') { + expect(file.cid).to.equal(expected['1.2MiB.txt'].cid) + expect(file.unixfs.fileSize()).to.equal(expected['1.2MiB.txt'].size) + } else if (file.path === 'pam/pum') { + expect(file.cid).to.equal(expected['pam/pum'].cid) + } else if (file.path === 'pam/1.2MiB.txt') { + expect(file.cid).to.equal(expected['1.2MiB.txt'].cid) + expect(file.unixfs.fileSize()).to.equal(expected['1.2MiB.txt'].size) + } else if (file.path === 'pam') { + expect(file.cid).to.equal(expected.pam.cid) + } else { + throw new Error(`Unexpected path ${file.path}`) } } }) it('will not write to disk if passed "onlyHash" option', async () => { const content = String(Math.random() + Date.now()) - const files = [] - - for await (const file of importer([{ + const files = await all(importer([{ path: content + '.txt', content: Buffer.from(content) }], ipld, { onlyHash: true - })) { - files.push(file) - } + })) const file = files[0] expect(file).to.exist() @@ -528,12 +536,10 @@ strategies.forEach((strategy) => { } } - for await (const _ of importer([{ // eslint-disable-line no-unused-vars + await all(importer([{ path: '1.2MiB.txt', content: bigFile - }], ipld, options)) { - // falala - } + }], ipld, options)) expect(options.progress.called).to.equal(true) expect(options.progress.args[0][0]).to.equal(maxChunkSize) @@ -568,12 +574,8 @@ strategies.forEach((strategy) => { shardSplitThreshold: 3 } - const files = [] - // Pass a copy of inputFiles, since the importer mutates them - for await (const file of importer(inputFiles.map(f => Object.assign({}, f)), ipld, options)) { - files.push(file) - } + const files = await all(importer(inputFiles.map(f => Object.assign({}, f)), ipld, options)) const file = files[0] expect(file).to.exist() From aee10a40f240f59440f3454ae95d2467c9f44015 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 May 2019 20:57:56 +0100 Subject: [PATCH 3/4] chore: update README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4824e3f..101f179 100644 --- a/README.md +++ b/README.md @@ -75,22 +75,22 @@ When run, metadata about DAGNodes in the created tree is printed until the root: ```js { cid: CID, // see https://github.com/multiformats/js-cid - path: '/tmp/foo/bar', + path: 'tmp/foo/bar', unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs } { cid: CID, // see https://github.com/multiformats/js-cid - path: '/tmp/foo/quxx', + path: 'tmp/foo/quxx', unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs } { cid: CID, // see https://github.com/multiformats/js-cid - path: '/tmp/foo', + path: 'tmp/foo', unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs } { cid: CID, // see https://github.com/multiformats/js-cid - path: '/tmp', + path: 'tmp', unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs } ``` From 1023e55a40bccbf5dd8be017f761e819556cca6d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 17 May 2019 11:24:21 +0200 Subject: [PATCH 4/4] chore: standardise error codes --- src/chunker/index.js | 2 +- src/chunker/rabin.js | 4 ++-- src/dag-builder/file/index.js | 2 +- src/dag-builder/validate-chunks.js | 2 +- src/dir-sharded.js | 10 ++++++---- src/flat-to-shard.js | 2 +- src/index.js | 5 +++++ src/tree-builder.js | 2 +- src/utils/persist.js | 4 ++++ test/builder-only-hash.spec.js | 2 ++ test/builder.spec.js | 2 ++ test/chunker-rabin-browser.spec.js | 2 +- test/importer.spec.js | 6 +++--- 13 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/chunker/index.js b/src/chunker/index.js index 5244679..ec2c494 100644 --- a/src/chunker/index.js +++ b/src/chunker/index.js @@ -11,7 +11,7 @@ module.exports = (type, source, options) => { const chunker = chunkers[type] if (!chunker) { - throw errCode(new Error(`Unknkown chunker named ${type}`), 'EUNKNOWNCHUNKER') + throw errCode(new Error(`Unknkown chunker named ${type}`), 'ERR_UNKNOWN_CHUNKER') } return chunker(source, options) diff --git a/src/chunker/rabin.js b/src/chunker/rabin.js index 8291f4c..ca130e6 100644 --- a/src/chunker/rabin.js +++ b/src/chunker/rabin.js @@ -10,10 +10,10 @@ module.exports = async function * rabinChunker (source, options) { createRabin = require('rabin') if (typeof createRabin !== 'function') { - throw errCode(new Error(`createRabin was not a function`), 'EUNSUPPORTED') + throw errCode(new Error(`createRabin was not a function`), 'ERR_UNSUPPORTED') } } catch (err) { - throw errCode(new Error(`Rabin chunker not available, it may have failed to install or not be supported on this platform`), 'EUNSUPPORTED') + throw errCode(new Error(`Rabin chunker not available, it may have failed to install or not be supported on this platform`), 'ERR_UNSUPPORTED') } } diff --git a/src/dag-builder/file/index.js b/src/dag-builder/file/index.js index a4ec634..c314210 100644 --- a/src/dag-builder/file/index.js +++ b/src/dag-builder/file/index.js @@ -119,7 +119,7 @@ const fileBuilder = async (file, source, ipld, options) => { const dagBuilder = dagBuilders[options.strategy] if (!dagBuilder) { - throw errCode(new Error(`Unknown importer build strategy name: ${options.strategy}`), 'EBADSTRATEGY') + throw errCode(new Error(`Unknown importer build strategy name: ${options.strategy}`), 'ERR_BAD_STRATEGY') } const roots = await all(dagBuilder(buildFile(source, ipld, options), reduce(file, ipld, options), options.builderOptions)) diff --git a/src/dag-builder/validate-chunks.js b/src/dag-builder/validate-chunks.js index 839461b..bf3037d 100644 --- a/src/dag-builder/validate-chunks.js +++ b/src/dag-builder/validate-chunks.js @@ -6,7 +6,7 @@ const errCode = require('err-code') async function * validateChunks (source) { for await (const content of source) { if (content.length === undefined) { - throw errCode(new Error('Content was invalid'), 'EINVALIDCONTENT') + throw errCode(new Error('Content was invalid'), 'ERR_INVALID_CONTENT') } if (typeof content === 'string' || content instanceof String) { diff --git a/src/dir-sharded.js b/src/dir-sharded.js index da7aeba..337fe30 100644 --- a/src/dir-sharded.js +++ b/src/dir-sharded.js @@ -80,9 +80,7 @@ class DirSharded extends Dir { } } -module.exports = (props, options) => { - return new DirSharded(props, options) -} +module.exports = DirSharded module.exports.hashFn = hashFn @@ -123,7 +121,11 @@ async function * flush (path, bucket, ipld, options) { const value = child.value if (!value.node) { - continue + if (value.cid) { + value.node = await ipld.get(value.cid) + } else { + continue + } } const label = labelPrefix + child.key diff --git a/src/flat-to-shard.js b/src/flat-to-shard.js index 8bd4879..aa7675e 100644 --- a/src/flat-to-shard.js +++ b/src/flat-to-shard.js @@ -29,7 +29,7 @@ module.exports = async function flatToShard (child, dir, threshold, options) { } async function convertToShard (oldDir, options) { - const newDir = DirSharded({ + const newDir = new DirSharded({ root: oldDir.root, dir: true, parent: oldDir.parent, diff --git a/src/index.js b/src/index.js index cd23b2f..7b5f1b6 100644 --- a/src/index.js +++ b/src/index.js @@ -41,6 +41,7 @@ const Options = struct({ strategy: struct.enum(['balanced', 'flat', 'trickle']), reduceSingleLeafToSelf: 'boolean?', codec: 'codec?', + format: 'codec?', hashAlg: 'hashAlg?', leafType: 'leafType?', cidVersion: 'number?', @@ -90,6 +91,10 @@ module.exports = async function * (source, ipld, options = {}) { opts.leafType = 'raw' } + if (options.format) { + options.codec = options.format + } + for await (const entry of treeBuilder(dagBuilder(source, ipld, opts), ipld, opts)) { yield { cid: entry.cid, diff --git a/src/tree-builder.js b/src/tree-builder.js index 7e19a65..8f48595 100644 --- a/src/tree-builder.js +++ b/src/tree-builder.js @@ -67,7 +67,7 @@ async function * treeBuilder (source, ipld, options) { if (tree) { if (!options.wrapWithDirectory) { if (tree.childCount() > 1) { - throw errCode(new Error('detected more than one root'), 'EMORETHANONEROOT') + throw errCode(new Error('detected more than one root'), 'ERR_MORE_THAN_ONE_ROOT') } const unwrapped = await first(tree.eachChildSeries()) diff --git a/src/utils/persist.js b/src/utils/persist.js index 6393837..110bb89 100644 --- a/src/utils/persist.js +++ b/src/utils/persist.js @@ -17,6 +17,10 @@ const persist = async (node, ipld, options) => { options.cidVersion = 1 } + if (options.format) { + options.codec = options.format + } + const format = mc[options.codec.toUpperCase().replace(/-/g, '_')] return ipld.put(node, format, options) diff --git a/test/builder-only-hash.spec.js b/test/builder-only-hash.spec.js index 2a08712..6514483 100644 --- a/test/builder-only-hash.spec.js +++ b/test/builder-only-hash.spec.js @@ -33,6 +33,8 @@ describe('builder: onlyHash', () => { progress: () => {}, leafType: 'file', reduceSingleLeafToSelf: true, + format: 'dag-pb', + hashAlg: 'sha2-256', wrap: true, chunkerOptions: { maxChunkSize: 1024 diff --git a/test/builder.spec.js b/test/builder.spec.js index 8b9914f..0b9f694 100644 --- a/test/builder.spec.js +++ b/test/builder.spec.js @@ -30,6 +30,8 @@ describe('builder', () => { chunker: 'fixed', leafType: 'file', reduceSingleLeafToSelf: true, + format: 'dag-pb', + hashAlg: 'sha2-256', progress: () => {}, chunkerOptions: { maxChunkSize: 262144 diff --git a/test/chunker-rabin-browser.spec.js b/test/chunker-rabin-browser.spec.js index a13eb8c..abac661 100644 --- a/test/chunker-rabin-browser.spec.js +++ b/test/chunker-rabin-browser.spec.js @@ -19,7 +19,7 @@ describe('chunker: rabin browser', () => { try { await all(chunker()) } catch (err) { - expect(err.code).to.equal('EUNSUPPORTED') + expect(err.code).to.equal('ERR_UNSUPPORTED') } }) }) diff --git a/test/importer.spec.js b/test/importer.spec.js index e23e651..1abb34c 100644 --- a/test/importer.spec.js +++ b/test/importer.spec.js @@ -267,7 +267,7 @@ strategies.forEach((strategy) => { }], ipld, options)) throw new Error('No error was thrown') } catch (err) { - expect(err.code).to.equal('EINVALIDCONTENT') + expect(err.code).to.equal('ERR_INVALID_CONTENT') } }) @@ -283,7 +283,7 @@ strategies.forEach((strategy) => { }], ipld, options)) throw new Error('No error was thrown') } catch (err) { - expect(err.code).to.equal('EINVALIDCONTENT') + expect(err.code).to.equal('ERR_INVALID_CONTENT') } }) @@ -317,7 +317,7 @@ strategies.forEach((strategy) => { throw new Error('No error was thrown') } catch (err) { - expect(err.code).to.equal('EMORETHANONEROOT') + expect(err.code).to.equal('ERR_MORE_THAN_ONE_ROOT') } })