From 8e5fc080699161e03cf01c9eeeeeff45a4d45c9b Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 11 Oct 2021 16:36:20 +1100 Subject: [PATCH] feat!: match go-ipfs@0.10 dag put options and semantics Fixes: https://github.com/ipfs/js-ipfs/issues/3914 Ref: https://github.com/ipfs/go-ipfs/blob/master/CHANGELOG.md#v0100-2021-09-30 --format and --input-enc have been replaced with --input-codec and --store-codec and mean something a little different. You now supply raw input and instruct the server which --input-codec that data is in which it will decode, then re-encode with --store-codec before storing it and providing you with the CID. We accept plain JavaScript objects to encode with --store-codec via the API here, defaulting to dag-cbor, and send that to the server as encoded bytes using that codec, to be stored using that codec. If you supply an --input-codec then we assume you're supplying raw, encoded bytes using that codec and we pass that directly on to the server to handle. --- packages/ipfs-core-types/src/dag/index.ts | 14 ++++- packages/ipfs-core/package.json | 2 +- packages/ipfs-core/src/components/dag/put.js | 23 +++++--- packages/ipfs-http-client/package.json | 2 +- packages/ipfs-http-client/src/dag/put.js | 25 +++++++-- packages/ipfs-http-client/test/dag.spec.js | 46 +++++++++------- .../ipfs-http-server/src/api/resources/dag.js | 52 +++++-------------- packages/ipfs/package.json | 2 +- 8 files changed, 93 insertions(+), 73 deletions(-) diff --git a/packages/ipfs-core-types/src/dag/index.ts b/packages/ipfs-core-types/src/dag/index.ts index 61a53b23c3..658e4f1e0a 100644 --- a/packages/ipfs-core-types/src/dag/index.ts +++ b/packages/ipfs-core-types/src/dag/index.ts @@ -107,6 +107,11 @@ export interface API { } export interface GetOptions extends AbortOptions, PreloadOptions { + /** + * The codec that the node will be encoded in (defaults to 'dag-json') + */ + inputCodec?: string + /** * An optional path within the DAG to resolve */ @@ -132,9 +137,14 @@ export interface GetResult { export interface PutOptions extends AbortOptions, PreloadOptions { /** - * The codec to use to create the CID (defaults to 'dag-cbor') + * The codec that the input object is encoded in (defaults to 'dag-json') + */ + inputCodec?: string + + /** + * The codec that the stored object will be encoded with (defaults to 'dag-cbor') */ - format?: string + storeCodec?: string /** * Multihash hashing algorithm to use (defaults to 'sha2-256') diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 766f28a550..38acceaeb9 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -136,7 +136,7 @@ "@types/rimraf": "^3.0.1", "aegir": "^35.0.3", "delay": "^5.0.0", - "go-ipfs": "0.9.1", + "go-ipfs": "0.10.0", "interface-blockstore-tests": "^1.0.0", "interface-ipfs-core": "^0.150.2", "ipfsd-ctl": "^10.0.3", diff --git a/packages/ipfs-core/src/components/dag/put.js b/packages/ipfs-core/src/components/dag/put.js index 6ee884abaf..fb2fe8ac9b 100644 --- a/packages/ipfs-core/src/components/dag/put.js +++ b/packages/ipfs-core/src/components/dag/put.js @@ -18,23 +18,32 @@ module.exports = ({ repo, codecs, hashers, preload }) => { const release = options.pin ? await repo.gcLock.readLock() : null try { - const codecName = options.format || 'dag-cbor' - const cidVersion = options.version != null ? options.version : (codecName === 'dag-pb' ? 0 : 1) - const codec = await codecs.getCodec(codecName) + const storeCodec = await codecs.getCodec(options.storeCodec || 'dag-cbor') + if (!storeCodec) { + throw new Error(`Unknown storeCodec ${options.storeCodec}, please configure additional BlockCodecs for this IPFS instance`) + } - if (!codec) { - throw new Error(`Unknown codec ${options.format}, please configure additional BlockCodecs for this IPFS instance`) + if (options.inputCodec) { + if (!(dagNode instanceof Uint8Array)) { + throw new Error('Can only inputCodec on raw bytes that can be decoded') + } + const inputCodec = await codecs.getCodec(options.inputCodec) + if (!inputCodec) { + throw new Error(`Unknown inputCodec ${options.inputCodec}, please configure additional BlockCodecs for this IPFS instance`) + } + dagNode = inputCodec.decode(dagNode) } + const cidVersion = options.version != null ? options.version : (options.storeCodec === 'dag-pb' ? 0 : 1) const hasher = await hashers.getHasher(options.hashAlg || 'sha2-256') if (!hasher) { throw new Error(`Unknown hash algorithm ${options.hashAlg}, please configure additional MultihashHashers for this IPFS instance`) } - const buf = codec.encode(dagNode) + const buf = storeCodec.encode(dagNode) const hash = await hasher.digest(buf) - const cid = CID.create(cidVersion, codec.code, hash) + const cid = CID.create(cidVersion, storeCodec.code, hash) await repo.blocks.put(cid, buf, { signal: options.signal diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index ffddd222ce..75145e1f43 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -69,7 +69,7 @@ "devDependencies": { "aegir": "^35.0.3", "delay": "^5.0.0", - "go-ipfs": "0.9.1", + "go-ipfs": "0.10.0", "ipfsd-ctl": "^10.0.3", "it-all": "^1.0.4", "it-first": "^1.0.4", diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index 678b930193..2f3ddc2fe1 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -23,14 +23,31 @@ module.exports = (codecs, options) => { */ const put = async (dagNode, options = {}) => { const settings = { - format: 'dag-cbor', + storeCodec: 'dag-cbor', hashAlg: 'sha2-256', - inputEnc: 'raw', ...options } - const codec = await codecs.getCodec(settings.format) - const serialized = codec.encode(dagNode) + let serialized + + if (settings.inputCodec) { + // if you supply an inputCodec, we assume you're passing in a raw, encoded + // block using that codec, so we'll just pass that on to the server and let + // it deal with the decode/encode/store cycle + if (!(dagNode instanceof Uint8Array)) { + throw new Error('Can only inputCodec on raw bytes that can be decoded') + } + serialized = dagNode + } else { + // if you don't supply an inputCodec, we assume you've passed in a JavaScript + // object you want to have encoded using storeCodec, so we'll prepare it for + // you if we have the codec + const storeCodec = await codecs.getCodec(settings.storeCodec) + serialized = storeCodec.encode(dagNode) + // now we have a serialized form, the server should be told to receive it + // in that format + settings.inputCodec = settings.storeCodec + } // allow aborting requests on body errors const controller = new AbortController() diff --git a/packages/ipfs-http-client/test/dag.spec.js b/packages/ipfs-http-client/test/dag.spec.js index e1823d9f9e..d0c4bbbbd5 100644 --- a/packages/ipfs-http-client/test/dag.spec.js +++ b/packages/ipfs-http-client/test/dag.spec.js @@ -5,6 +5,7 @@ const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string') const { expect } = require('aegir/utils/chai') +const { CID } = require('multiformats/cid') const dagPb = require('@ipld/dag-pb') const dagCbor = require('@ipld/dag-cbor') const raw = require('multiformats/codecs/raw') @@ -23,25 +24,25 @@ describe('.dag', function () { after(() => f.clean()) - it('should be able to put and get a DAG node with format dag-pb', async () => { + it('should be able to put and get a DAG node with dag-pb codec', async () => { const data = uint8ArrayFromString('some data') const node = { Data: data, Links: [] } - const cid = await ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256', cidVersion: 0 }) + const cid = await ipfs.dag.put(node, { storeCodec: 'dag-pb', hashAlg: 'sha2-256' }) expect(cid.code).to.equal(dagPb.code) - expect(cid.toString(base58btc)).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr') + expect(cid.toV0().toString()).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr') const result = await ipfs.dag.get(cid) expect(result.value.Data).to.deep.equal(data) }) - it('should be able to put and get a DAG node with format dag-cbor', async () => { + it('should be able to put and get a DAG node with dag-cbor codec', async () => { const cbor = { foo: 'dag-cbor-bar' } - const cid = await ipfs.dag.put(cbor, { format: 'dag-cbor', hashAlg: 'sha2-256' }) + const cid = await ipfs.dag.put(cbor, { storeCodec: 'dag-cbor', hashAlg: 'sha2-256' }) expect(cid.code).to.equal(dagCbor.code) expect(cid.toString(base32)).to.equal('bafyreic6f672hnponukaacmk2mmt7vs324zkagvu4hcww6yba6kby25zce') @@ -51,9 +52,9 @@ describe('.dag', function () { expect(result.value).to.deep.equal(cbor) }) - it('should be able to put and get a DAG node with format raw', async () => { + it('should be able to put and get a DAG node with raw codec', async () => { const node = uint8ArrayFromString('some data') - const cid = await ipfs.dag.put(node, { format: 'raw', hashAlg: 'sha2-256' }) + const cid = await ipfs.dag.put(node, { storeCodec: 'raw', hashAlg: 'sha2-256' }) expect(cid.code).to.equal(raw.code) expect(cid.toString(base32)).to.equal('bafkreiata6mq425fzikf5m26temcvg7mizjrxrkn35swuybmpah2ajan5y') @@ -71,19 +72,28 @@ describe('.dag', function () { await expect(ipfs.dag.get(cid)).to.eventually.be.rejectedWith(/No codec found/) }) - it('should error when putting node with esoteric format', () => { + it('should error when putting node with esoteric codec', () => { const node = uint8ArrayFromString('some data') - return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/No codec found/) + return expect(ipfs.dag.put(node, { storeCodec: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/No codec found/) }) - it('should attempt to load an unsupported format', async () => { - let askedToLoadFormat + it('should attempt to load an unsupported codec', async () => { + const node = uint8ArrayFromString('blob 9\0some data') + // we don't support git-raw in the HTTP client, but inputCodec and a Uint8Array should make + // the raw data pass through to go-ipfs, which does talk git-raw + const cid = await ipfs.dag.put(node, { inputCodec: 'git-raw', storeCodec: 'git-raw', hashAlg: 'sha1' }) + expect(cid.code).to.equal(0x78) + expect(cid.toString(base32)).to.equal('baf4bcfd4azdl7vj4d4hnix75qfld6mabo4l4uwa') + }) + + it('should attempt to load an unsupported codec', async () => { + let askedToLoadCodec const ipfs2 = ipfsHttpClient.create({ url: `http://${ipfs.apiHost}:${ipfs.apiPort}`, ipld: { - loadCodec: (format) => { - askedToLoadFormat = format === 'git-raw' + loadCodec: (codec) => { + askedToLoadCodec = codec === 'boop' return { encode: (buf) => buf } @@ -94,12 +104,12 @@ describe('.dag', function () { const node = uint8ArrayFromString('some data') // error is from go-ipfs, this means the client serialized it ok - await expect(ipfs2.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/no parser for format "git-raw"/) + await expect(ipfs2.dag.put(node, { storeCodec: 'boop', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/unknown multicodec: "boop"/) - expect(askedToLoadFormat).to.be.true() + expect(askedToLoadCodec).to.be.true() }) - it('should allow formats to be specified without overwriting others', async () => { + it('should allow codecs to be specified without overwriting others', async () => { const ipfs2 = ipfsHttpClient.create({ url: `http://${ipfs.apiHost}:${ipfs.apiPort}`, ipld: { @@ -116,7 +126,7 @@ describe('.dag', function () { hello: 'world' } const cid1 = await ipfs2.dag.put(dagCborNode, { - format: 'dag-cbor', + storeCodec: 'dag-cbor', hashAlg: 'sha2-256' }) @@ -125,7 +135,7 @@ describe('.dag', function () { Links: [] } const cid2 = await ipfs2.dag.put(dagPbNode, { - format: 'dag-pb', + storeCodec: 'dag-pb', hashAlg: 'sha2-256' }) diff --git a/packages/ipfs-http-server/src/api/resources/dag.js b/packages/ipfs-http-server/src/api/resources/dag.js index 1e3feef0bc..c66e1bbacb 100644 --- a/packages/ipfs-http-server/src/api/resources/dag.js +++ b/packages/ipfs-http-server/src/api/resources/dag.js @@ -129,7 +129,8 @@ exports.put = { throw Boom.badRequest("File argument 'object data' is required") } - const enc = request.query.inputEncoding + const inputCodec = request.query.inputCodec + const storeCodec = request.query.storeCodec if (!request.headers['content-type']) { throw Boom.badRequest("File argument 'object data' is required") @@ -149,42 +150,19 @@ exports.put = { throw Boom.badRequest("File argument 'object data' is required") } - let format = request.query.format + const cidVersion = format === 'dag-pb' && request.query.hashAlg === 'sha2-256' ? request.query.version : 1 - if (format === 'cbor') { - format = 'dag-cbor' - } - - let node + const cid = await request.server.app.ipfs.block.put(data, { + version: cidVersion, + storeCodec, + inputCodec, + mhtype: request.query.hash + }) - if (format === 'raw') { - node = data - } else if (enc === 'json') { - try { - node = JSON.parse(data.toString()) - } catch (err) { - throw Boom.badRequest('Failed to parse the JSON: ' + err) - } - } else { - // the node is an uncommon format which the client should have - // serialized so add it to the block store and fetch it deserialized - // before continuing - const cidVersion = format === 'dag-pb' && request.query.hashAlg === 'sha2-256' ? request.query.version : 1 - - const cid = await request.server.app.ipfs.block.put(data, { - version: cidVersion, - format, - mhtype: request.query.hash - }) - - const { - value - } = await request.server.app.ipfs.dag.get(cid) - node = value - } + const { value } = await request.server.app.ipfs.dag.get(cid) return { - node, + node: value, format, hashAlg: request.query.hash } @@ -196,18 +174,14 @@ exports.put = { stripUnknown: true }, query: Joi.object().keys({ - format: Joi.string().default('cbor'), - inputEncoding: Joi.string().default('json'), + storeCodec: Joi.string().default('dag-cbor'), + inputCodec: Joi.string().default('dag-json'), pin: Joi.boolean().default(false), hash: Joi.string().default('sha2-256'), cidBase: Joi.string().default('base32'), version: Joi.number().integer().valid(0, 1).default(1), timeout: Joi.timeout() }) - .rename('input-enc', 'inputEncoding', { - override: true, - ignoreUndefined: true - }) .rename('cid-base', 'cidBase', { override: true, ignoreUndefined: true diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index bd29b3fd45..af924429c0 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -51,7 +51,7 @@ "assert": "^2.0.0", "cross-env": "^7.0.0", "electron-webrtc": "^0.3.0", - "go-ipfs": "0.9.1", + "go-ipfs": "0.10.0", "interface-ipfs-core": "^0.150.2", "ipfs-client": "^0.6.4", "ipfs-core-types": "^0.7.1",