diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index b164cef339..1cba93d31b 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -171,7 +171,7 @@ An optional object which may have the following keys: | hashAlg | `String` | `'sha2-256'` | multihash hashing algorithm to use | | onlyHash | `boolean` | `false` | If true, will not add blocks to the blockstore | | pin | `boolean` | `true` | pin this object when adding | -| progress | function | `undefined` | a function that will be called with the byte length of chunks as a file is added to ipfs | +| progress | function | `undefined` | a function that will be called with the number of bytes added as a file is added to ipfs and the path of the file being added | | rawLeaves | `boolean` | `false` | if true, DAG leaves will contain raw file data and not be wrapped in a protobuf | | trickle | `boolean` | `false` | if true will use the [trickle DAG](https://godoc.org/github.com/ipsn/go-ipfs/gxlibs/github.com/ipfs/go-unixfs/importer/trickle) format for DAG generation | | wrapWithDirectory | `boolean` | `false` | Adds a wrapping node around the content | @@ -252,7 +252,7 @@ An optional object which may have the following keys: | hashAlg | `String` | `'sha2-256'` | multihash hashing algorithm to use | | onlyHash | `boolean` | `false` | If true, will not add blocks to the blockstore | | pin | `boolean` | `true` | pin this object when adding | -| progress | function | `undefined` | a function that will be called with the number of bytes added as a file is added to ipfs and the name of the file being added | +| progress | function | `undefined` | a function that will be called with the number of bytes added as a file is added to ipfs and the path of the file being added | | rawLeaves | `boolean` | `false` | if true, DAG leaves will contain raw file data and not be wrapped in a protobuf | | shardSplitThreshold | `Number` | `1000` | Directories with more than this number of files will be created as HAMT-sharded directories | | trickle | `boolean` | `false` | if true will use the [trickle DAG](https://godoc.org/github.com/ipsn/go-ipfs/gxlibs/github.com/ipfs/go-unixfs/importer/trickle) format for DAG generation | diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index 46887d3af7..bb3ea5a145 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -140,6 +140,7 @@ module.exports = (common, options) => { }) const emptyDir = (name) => ({ path: `test-folder/${name}` }) + const progressSizes = {} const dirs = [ content('pp.txt'), @@ -152,24 +153,46 @@ module.exports = (common, options) => { emptyDir('files/empty') ] - const total = dirs.reduce((i, entry) => { - return i + (entry.content ? entry.content.length : 0) - }, 0) + const total = dirs.reduce((acc, curr) => { + if (curr.content) { + acc[curr.path] = curr.content.length + } + + return acc + }, {}) - let progCalled = false - let accumProgress = 0 - const handler = (p) => { - progCalled = true - accumProgress += p + const handler = (bytes, path) => { + progressSizes[path] = bytes } const root = await last(ipfs.addAll(dirs, { progress: handler })) - expect(progCalled).to.be.true() - expect(accumProgress).to.be.at.least(total) + expect(progressSizes).to.deep.equal(total) expect(root.path).to.equal('test-folder') expect(root.cid.toString()).to.equal(fixtures.directory.cid) }) + it('should receive progress path as empty string when adding content without paths', async function () { + const content = (name) => fixtures.directory.files[name] + const progressSizes = {} + + const dirs = [ + content('pp.txt'), + content('holmes.txt'), + content('jungle.txt') + ] + + const total = { + '': dirs.reduce((acc, curr) => acc + curr.length, 0) + } + + const handler = (bytes, path) => { + progressSizes[path] = bytes + } + + await drain(ipfs.addAll(dirs, { progress: handler })) + expect(progressSizes).to.deep.equal(total) + }) + it('should receive file name from progress event', async () => { const receivedNames = [] function handler (p, name) { diff --git a/packages/ipfs-core/src/components/add-all/index.js b/packages/ipfs-core/src/components/add-all/index.js index dd90d39af1..4798616df7 100644 --- a/packages/ipfs-core/src/components/add-all/index.js +++ b/packages/ipfs-core/src/components/add-all/index.js @@ -60,13 +60,19 @@ module.exports = ({ block, gcLock, preload, pin, options: constructorOptions }) delete opts.trickle + const totals = {} + if (opts.progress) { - let total = 0 const prog = opts.progress - opts.progress = (bytes, fileName) => { - total += bytes - prog(total, fileName) + opts.progress = (bytes, path) => { + if (!totals[path]) { + totals[path] = 0 + } + + totals[path] += bytes + + prog(totals[path], path) } } @@ -84,7 +90,12 @@ module.exports = ({ block, gcLock, preload, pin, options: constructorOptions }) const releaseLock = await gcLock.readLock() try { - yield * iterator + for await (const added of iterator) { + // do not keep file totals around forever + delete totals[added.path] + + yield added + } } finally { releaseLock() } @@ -179,8 +190,7 @@ function pinFile (pin, opts) { * @property {boolean} [onlyHash=false] - If true, will not add blocks to the * blockstore. * @property {boolean} [pin=true] - Pin this object when adding. - * @property {(bytes:number, fileName:string) => void} [progress] - A function that will be - * called with the number of bytes added as a file is added to ipfs and the name of the file being added. + * @property {(bytes:number, path:string) => void} [progress] - a function that will be called with the number of bytes added as a file is added to ipfs and the path of the file being added * @property {boolean} [rawLeaves=false] - If true, DAG leaves will contain raw * file data and not be wrapped in a protobuf. * @property {number} [shardSplitThreshold=1000] - Directories with more than this diff --git a/packages/ipfs-core/src/components/add.js b/packages/ipfs-core/src/components/add.js index 130885a112..7334071e7c 100644 --- a/packages/ipfs-core/src/components/add.js +++ b/packages/ipfs-core/src/components/add.js @@ -25,7 +25,7 @@ module.exports = ({ addAll }) => { * @property {string} [hashAlg] - multihash hashing algorithm to use (default: `'sha2-256'`) * @property {boolean} [onlyHash] - If true, will not add blocks to the blockstore (default: `false`) * @property {boolean} [pin] - pin this object when adding (default: `true`) - * @property {Function} [progress] - a function that will be called with the byte length of chunks as a file is added to ipfs (default: `undefined`) + * @property {(bytes:number, path:string) => void} [progress] - a function that will be called with the number of bytes added as a file is added to ipfs and the path of the file being added * @property {boolean} [rawLeaves] - if true, DAG leaves will contain raw file data and not be wrapped in a protobuf (default: `false`) * @property {boolean} [trickle] - if true will use the [trickle DAG](https://godoc.org/github.com/ipsn/go-ipfs/gxlibs/github.com/ipfs/go-unixfs/importer/trickle) format for DAG generation (default: `false`) * @property {boolean} [wrapWithDirectory] - Adds a wrapping node around the content (default: `false`) diff --git a/packages/ipfs-http-server/src/api/resources/files-regular.js b/packages/ipfs-http-server/src/api/resources/files-regular.js index af1bf6f400..b3d27bc688 100644 --- a/packages/ipfs-http-server/src/api/resources/files-regular.js +++ b/packages/ipfs-http-server/src/api/resources/files-regular.js @@ -260,7 +260,7 @@ exports.add = { multipart(request), async function * (source) { for await (const entry of source) { - currentFileName = entry.name || 'unknown' + currentFileName = entry.name || '' if (entry.type === 'file') { filesParsed = true