From 51899e1c67c0bdd0579d8f5bf8c3a260ea72e627 Mon Sep 17 00:00:00 2001 From: nginnever Date: Thu, 5 May 2016 06:38:05 -0700 Subject: [PATCH] add, cat, get core + cli offline --- package.json | 3 +- src/cli/commands/files/add.js | 126 ++++++++++++++++-- src/cli/commands/files/cat.js | 48 +++++++ src/cli/commands/files/get.js | 69 ++++++++++ src/core/index.js | 54 +++++++- src/core/init.js | 45 ++++++- test/cli-tests/test-commands.js | 2 +- test/core-tests/test-files.js | 64 +++++++++ test/core-tests/test-init-node.js | 14 +- ...dc5e844ead5f91f12858b021eba45768b4c0e.data | 3 + 10 files changed, 401 insertions(+), 27 deletions(-) create mode 100644 src/cli/commands/files/cat.js create mode 100644 src/cli/commands/files/get.js create mode 100644 test/core-tests/test-files.js create mode 100644 test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data diff --git a/package.json b/package.json index 1cc32fe4c0..f88d11fc46 100644 --- a/package.json +++ b/package.json @@ -65,10 +65,10 @@ "ipfs-api": "^3.0.1", "ipfs-block": "^0.3.0", "ipfs-block-service": "^0.3.0", - "ipfs-data-importing": "^0.3.3", "ipfs-merkle-dag": "^0.5.0", "ipfs-multipart": "^0.1.0", "ipfs-repo": "^0.7.1", + "ipfs-unixfs-engine": "^0.5.0", "joi": "^8.0.2", "libp2p-ipfs": "^0.3.3", "lodash.get": "^4.2.1", @@ -77,6 +77,7 @@ "peer-book": "0.1.0", "peer-id": "^0.6.6", "peer-info": "^0.6.2", + "readable-stream": "^1.1.13", "ronin": "^0.3.11", "temp": "^0.8.3" }, diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 5a0fd79138..21cca05fad 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -1,11 +1,16 @@ 'use strict' const Command = require('ronin').Command -const IPFS = require('../../../core') +const utils = require('../../utils') const debug = require('debug') const log = debug('cli:version') log.error = debug('cli:version:error') const bs58 = require('bs58') +const Readable = require('stream').Readable +const fs = require('fs') +const async = require('async') +const pathj = require('path') +const glob = require('glob') module.exports = Command.extend({ desc: 'Add a file to IPFS using the UnixFS data format', @@ -19,15 +24,120 @@ module.exports = Command.extend({ }, run: (recursive, path) => { - var node = new IPFS() - path = process.cwd() + '/' + path - node.files.add(path, { - recursive: recursive - }, (err, stats) => { + let s + let rs + + if (!path) { + throw new Error('Error: Argument \'path\' is required') + } + + s = fs.statSync(path) + + if (s.isDirectory() && recursive === false) { + throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories') + } + if (path === '.' && recursive === true) { + path = process.cwd() + s = fs.statSync(process.cwd()) + } else if (path === '.' && recursive === false) { + s = fs.statSync(process.cwd()) + if (s.isDirectory()) { + throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories') + } + } + + glob(pathj.join(path, '/**/*'), (err, res) => { if (err) { - return console.log(err) + throw err } - console.log('added', bs58.encode(stats.Hash).toString(), stats.Name) + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + // var files = [] + if (utils.isDaemonOn()) { + throw new Error('on-line mode is not supported yet') + /* if (res.length !== 0) { + const index = path.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + rs = new Readable() + const addPath = element.substring(index + 1, element.length) + if (fs.statSync(element).isDirectory()) { + callback() + } else { + const buffered = fs.readFileSync(element) + rs.push(buffered) + rs.push(null) + const filePair = {path: addPath, content: rs} + files.push(filePair) + callback() + } + }, (err) => { + if (err) { + throw err + } + ipfs.add(files, (err, res) => { + if (err) { + throw err + } + res.forEach((goRes) => { + console.log('added', goRes.Hash, goRes.Name) + }) + }) + }) + } else { + rs = new Readable() + const buffered = fs.readFileSync(path) + rs.push(buffered) + rs.push(null) + path = path.substring(path.lastIndexOf('/') + 1, path.length) + const filePair = {path: path, content: rs} + files.push(filePair) + ipfs.add(files, (err, res) => { + if (err) { + throw err + } + console.log('added', res[0].Hash, res[0].Name) + }) + } + return */ + } + const i = ipfs.files.add() + i.on('data', (file) => { + console.log('added', bs58.encode(file.multihash).toString(), file.path) + }) + if (res.length !== 0) { + const index = path.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + rs = new Readable() + const addPath = element.substring(index + 1, element.length) + if (fs.statSync(element).isDirectory()) { + callback() + } else { + const buffered = fs.readFileSync(element) + rs.push(buffered) + rs.push(null) + const filePair = {path: addPath, stream: rs} + i.write(filePair) + callback() + } + }, (err) => { + if (err) { + throw err + } + i.end() + }) + } else { + rs = new Readable() + const buffered = fs.readFileSync(path) + path = path.substring(path.lastIndexOf('/') + 1, path.length) + rs.push(buffered) + rs.push(null) + const filePair = {path: path, stream: rs} + i.write(filePair) + i.end() + } + }) }) } }) diff --git a/src/cli/commands/files/cat.js b/src/cli/commands/files/cat.js new file mode 100644 index 0000000000..5c374a604e --- /dev/null +++ b/src/cli/commands/files/cat.js @@ -0,0 +1,48 @@ +'use strict' + +const Command = require('ronin').Command +const debug = require('debug') +const utils = require('../../utils') +const log = debug('cli:files') +log.error = debug('cli:files:error') + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + options: {}, + + run: (path, options) => { + if (!path) { + throw new Error("Argument 'path' is required") + } + if (!options) { + options = {} + } + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + if (utils.isDaemonOn()) { + throw new Error('on-line mode is not supported yet') + /* ipfs.cat(path, (err, res) => { + if (err) { + throw (err) + } + console.log(res.toString()) + }) + return */ + } + + ipfs.files.cat(path, (err, res) => { + if (err) { + throw (err) + } + if (res) { + res.on('file', (data) => { + data.stream.pipe(process.stdout) + }) + } + }) + }) + } +}) diff --git a/src/cli/commands/files/get.js b/src/cli/commands/files/get.js new file mode 100644 index 0000000000..b33e57541e --- /dev/null +++ b/src/cli/commands/files/get.js @@ -0,0 +1,69 @@ +'use strict' + +const Command = require('ronin').Command +const debug = require('debug') +const utils = require('../../utils') +const log = debug('cli:files') +log.error = debug('cli:files:error') +var fs = require('fs') +const pathj = require('path') + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + options: {}, + + run: (path, options) => { + let dir + let filepath + let ws + + if (!path) { + throw new Error("Argument 'path' is required") + } + if (!options) { + options = {} + dir = process.cwd() + } else { + if (options.slice(-1) !== '/') { + options += '/' + } + dir = options + } + + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + ipfs.files.get(path, (err, data) => { + if (err) { + throw err + } + data.on('file', (data) => { + if (data.path.lastIndexOf('/') === -1) { + filepath = data.path + if (data.dir === false) { + ws = fs.createWriteStream(pathj.join(dir, data.path)) + data.stream.pipe(ws) + } else { + try { + fs.mkdirSync(pathj.join(dir, data.path)) + } catch (err) { + throw err + } + } + } else { + filepath = data.path.substring(0, data.path.lastIndexOf('/') + 1) + try { + fs.mkdirSync(pathj.join(dir, filepath)) + } catch (err) { + throw err + } + ws = fs.createWriteStream(pathj.join(dir, data.path)) + data.stream.pipe(ws) + } + }) + }) + }) + } +}) diff --git a/src/core/index.js b/src/core/index.js index 474ade13fa..b9c88c2ed1 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -9,11 +9,13 @@ const DAGService = mDAG.DAGService const peerId = require('peer-id') const PeerInfo = require('peer-info') const multiaddr = require('multiaddr') -const importer = require('ipfs-data-importing').import +const Importer = require('ipfs-unixfs-engine').importer +const Exporter = require('ipfs-unixfs-engine').exporter const libp2p = require('libp2p-ipfs') const init = require('./init') const IPFSRepo = require('ipfs-repo') const PeerBook = require('peer-book') +const UnixFS = require('ipfs-unixfs') exports = module.exports = IPFS @@ -393,10 +395,52 @@ function IPFS (repo) { } this.files = { - add: (path, options, callback) => { - options.path = path - options.dagService = dagS - importer(options, callback) + add: (arr, callback) => { + if (typeof arr === 'function') { + callback = arr + arr = undefined + } + if (callback === undefined) { + callback = function noop () {} + } + if (arr === undefined) { + return new Importer(dagS) + } + + const i = new Importer(dagS) + const res = [] + + i.on('data', (info) => { + res.push(info) + }) + + i.on('end', () => { + callback(null, res) + }) + + arr.forEach((tuple) => { + i.write(tuple) + }) + + i.end() + }, + cat: (hash, callback) => { + dagS.get(hash, (err, fetchedNode) => { + if (err) { + return callback(err, null) + } + const data = UnixFS.unmarshal(fetchedNode.data) + if (data.type === 'directory') { + callback('This dag node is a directory', null) + } else { + const exportEvent = Exporter(hash, dagS) + callback(null, exportEvent) + } + }) + }, + get: (hash, callback) => { + var exportFile = Exporter(hash, dagS) + callback(null, exportFile) } } } diff --git a/src/core/init.js b/src/core/init.js index 76fa796a19..138528d2c2 100644 --- a/src/core/init.js +++ b/src/core/init.js @@ -4,6 +4,10 @@ const peerId = require('peer-id') const BlockService = require('ipfs-block-service') const DagService = require('ipfs-merkle-dag').DAGService const path = require('path') +const glob = require('glob') +const async = require('async') +const Readable = require('stream').Readable +const fs = require('fs') module.exports = (repo, opts, callback) => { opts = opts || {} @@ -63,17 +67,48 @@ module.exports = (repo, opts, callback) => { return doneImport(null) } - const importer = require('ipfs-data-importing') + const Importer = require('ipfs-unixfs-engine').importer const blocks = new BlockService(repo) const dag = new DagService(blocks) const initDocsPath = path.join(__dirname, '../init-files/init-docs') - importer.import(initDocsPath, dag, { - recursive: true - }, doneImport) + const i = new Importer(dag) + i.on('data', (file) => { + }) + + glob(path.join(initDocsPath, '/**/*'), (err, res) => { + if (err) { + throw err + } + const index = __dirname.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + const addPath = element.substring(index + 1, element.length) + if (fs.statSync(element).isDirectory()) { + callback() + } else { + const buffered = fs.readFileSync(element) + var rs = new Readable() + rs.push(buffered) + rs.push(null) + const filePair = {path: addPath, stream: rs} + i.write(filePair) + callback() + } + }, (err) => { + if (err) { + throw err + } + i.end() + return + }) + }) + + i.on('end', () => { + doneImport(null) + }) - function doneImport (err, stat) { + function doneImport (err) { if (err) { return callback(err) } // All finished! diff --git a/test/cli-tests/test-commands.js b/test/cli-tests/test-commands.js index aa8da00762..efb656d58a 100644 --- a/test/cli-tests/test-commands.js +++ b/test/cli-tests/test-commands.js @@ -10,7 +10,7 @@ describe('commands', () => { .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout.length).to.equal(45) + expect(stdout.length).to.equal(47) done() }) }) diff --git a/test/core-tests/test-files.js b/test/core-tests/test-files.js new file mode 100644 index 0000000000..f134a61d1e --- /dev/null +++ b/test/core-tests/test-files.js @@ -0,0 +1,64 @@ +/* eslint-env mocha */ +'use strict' + +const bl = require('bl') +const expect = require('chai').expect +const Readable = require('stream').Readable +const bs58 = require('bs58') + +const IPFS = require('../../src/core') + +describe('files', () => { + var ipfs + + before((done) => { + ipfs = new IPFS(require('./repo-path')) + ipfs.load(done) + }) + + it('add', (done) => { + const buffered = new Buffer('some data') + var rs = new Readable() + rs.push(buffered) + rs.push(null) + var arr = [] + const filePair = {path: 'data.txt', stream: rs} + arr.push(filePair) + ipfs.files.add(arr, (err, res) => { + expect(err).to.not.exist + expect(res[0].path).to.equal('data.txt') + expect(res[0].size).to.equal(17) + expect(bs58.encode(res[0].multihash).toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') + done() + }) + }) + + it('cat', (done) => { + const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + ipfs.files.cat(hash, (err, res) => { + expect(err).to.not.exist + res.on('file', (data) => { + data.stream.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.equal('hello world\n') + done() + })) + }) + }) + }) + + it('get', (done) => { + // TODO create non-trival get test + const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + ipfs.files.get(hash, (err, res) => { + expect(err).to.not.exist + res.on('file', (data) => { + data.stream.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.equal('hello world\n') + done() + })) + }) + }) + }) +}) diff --git a/test/core-tests/test-init-node.js b/test/core-tests/test-init-node.js index 1e05d61b5e..ee65c3e83e 100644 --- a/test/core-tests/test-init-node.js +++ b/test/core-tests/test-init-node.js @@ -24,14 +24,14 @@ describe('init (Node.js specific)', function () { it('init docs are written', (done) => { ipfs.init({ bits: 64 }, (err) => { expect(err).to.not.exist - - // Check for default assets var multihash = new Buffer('12205e7c3ce237f936c76faf625e90f7751a9f5eeb048f59873303c215e9cce87599', 'hex') - ipfs.object.get(multihash, {}, (err, node) => { - expect(err).to.not.exist - expect(node.links).to.exist - done() - }) + setTimeout(() => { + ipfs.object.get(multihash, {}, (err, node) => { + expect(err).to.not.exist + expect(node.links).to.exist + done() + }) + }, 1000) }) }) diff --git a/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data b/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data new file mode 100644 index 0000000000..2965d1c457 --- /dev/null +++ b/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data @@ -0,0 +1,3 @@ + + hello world + \ No newline at end of file