From b7b0b3f8f30f3c681c3c59389254ddb6724858d7 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 11 Oct 2021 16:36:20 +1100 Subject: [PATCH] feat!: update http-client DAG API to match go-ipfs@0.10 Fixes: https://github.com/ipfs/js-ipfs/issues/3914 Ref: https://github.com/ipfs/go-ipfs/blob/master/CHANGELOG.md#v0100-2021-09-30 --format and --input-enc have been replaced with --input-codec and --store-codec and mean something a little different. You now supply raw input and instruct the server which --input-codec that data is in which it will decode, then re-encode with --store-codec before storing it and providing you with the CID. We accept plain JavaScript objects to encode with --store-codec via the API here, defaulting to dag-cbor, and send that to the server as encoded bytes using that codec, to be stored using that codec. If you supply an --input-codec then we assume you're supplying raw, encoded bytes using that codec and we pass that directly on to the server to handle. --- packages/ipfs-core-types/src/dag/index.ts | 7 +++- packages/ipfs-http-client/package.json | 3 +- packages/ipfs-http-client/src/dag/put.js | 25 ++++++++++-- packages/ipfs-http-client/src/index.js | 3 +- packages/ipfs-http-client/test/dag.spec.js | 44 +++++++++++++--------- 5 files changed, 57 insertions(+), 25 deletions(-) diff --git a/packages/ipfs-core-types/src/dag/index.ts b/packages/ipfs-core-types/src/dag/index.ts index 61a53b23c3..cf27e69398 100644 --- a/packages/ipfs-core-types/src/dag/index.ts +++ b/packages/ipfs-core-types/src/dag/index.ts @@ -132,7 +132,12 @@ export interface GetResult { export interface PutOptions extends AbortOptions, PreloadOptions { /** - * The codec to use to create the CID (defaults to 'dag-cbor') + * The codec that the input object is encoded with (defaults to 'dag-json') + */ + inputCodec?: string + + /** + * The codec that the stored object will be encoded with (defaults to 'dag-cbor') */ format?: string diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 3c5a563d74..d48eb2de9c 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -55,6 +55,7 @@ }, "dependencies": { "@ipld/dag-cbor": "^6.0.5", + "@ipld/dag-json": "^8.0.1", "@ipld/dag-pb": "^2.1.3", "abort-controller": "^3.0.0", "any-signal": "^2.1.2", @@ -76,7 +77,7 @@ "devDependencies": { "aegir": "^35.1.1", "delay": "^5.0.0", - "go-ipfs": "0.9.1", + "go-ipfs": "0.10.0", "ipfsd-ctl": "^10.0.4", "it-all": "^1.0.4", "it-first": "^1.0.4", diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index 6192e1750a..71701942b9 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -21,14 +21,31 @@ export const createPut = (codecs, options) => { */ const put = async (dagNode, options = {}) => { const settings = { - format: 'dag-cbor', + storeCodec: 'dag-cbor', hashAlg: 'sha2-256', - inputEnc: 'raw', ...options } - const codec = await codecs.getCodec(settings.format) - const serialized = codec.encode(dagNode) + let serialized + + if (settings.inputCodec) { + // if you supply an inputCodec, we assume you're passing in a raw, encoded + // block using that codec, so we'll just pass that on to the server and let + // it deal with the decode/encode/store cycle + if (!(dagNode instanceof Uint8Array)) { + throw new Error('Can only inputCodec on raw bytes that can be decoded') + } + serialized = dagNode + } else { + // if you don't supply an inputCodec, we assume you've passed in a JavaScript + // object you want to have encoded using storeCodec, so we'll prepare it for + // you if we have the codec + const storeCodec = await codecs.getCodec(settings.storeCodec) + serialized = storeCodec.encode(dagNode) + // now we have a serialized form, the server should be told to receive it + // in that format + settings.inputCodec = settings.storeCodec + } // allow aborting requests on body errors const controller = new AbortController() diff --git a/packages/ipfs-http-client/src/index.js b/packages/ipfs-http-client/src/index.js index 4a8ab2ea92..bb7d055f72 100644 --- a/packages/ipfs-http-client/src/index.js +++ b/packages/ipfs-http-client/src/index.js @@ -5,6 +5,7 @@ import { Multicodecs } from 'ipfs-core-utils/multicodecs' import { Multihashes } from 'ipfs-core-utils/multihashes' import * as dagPB from '@ipld/dag-pb' import * as dagCBOR from '@ipld/dag-cbor' +import * as dagJSON from '@ipld/dag-json' import { identity } from 'multiformats/hashes/identity' import { bases, hashes, codecs } from 'multiformats/basics' import { createBitswap } from './bitswap/index.js' @@ -79,7 +80,7 @@ export function create (options = {}) { /** @type {BlockCodec[]} */ const blockCodecs = Object.values(codecs); - [dagPB, dagCBOR, id].concat((options.ipld && options.ipld.codecs) || []).forEach(codec => blockCodecs.push(codec)) + [dagPB, dagCBOR, dagJSON, id].concat((options.ipld && options.ipld.codecs) || []).forEach(codec => blockCodecs.push(codec)) const multicodecs = new Multicodecs({ codecs: blockCodecs, diff --git a/packages/ipfs-http-client/test/dag.spec.js b/packages/ipfs-http-client/test/dag.spec.js index 24c56b2269..346828fdc0 100644 --- a/packages/ipfs-http-client/test/dag.spec.js +++ b/packages/ipfs-http-client/test/dag.spec.js @@ -6,7 +6,6 @@ import { expect } from 'aegir/utils/chai.js' import * as dagPB from '@ipld/dag-pb' import * as dagCBOR from '@ipld/dag-cbor' import * as raw from 'multiformats/codecs/raw' -import { base58btc } from 'multiformats/bases/base58' import { base32 } from 'multiformats/bases/base32' import { create as httpClient } from '../src/index.js' import { factory } from './utils/factory.js' @@ -22,25 +21,25 @@ describe('.dag', function () { after(() => f.clean()) - it('should be able to put and get a DAG node with format dag-pb', async () => { + it('should be able to put and get a DAG node with dag-pb codec', async () => { const data = uint8ArrayFromString('some data') const node = { Data: data, Links: [] } - const cid = await ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256', cidVersion: 0 }) + const cid = await ipfs.dag.put(node, { storeCodec: 'dag-pb', hashAlg: 'sha2-256' }) expect(cid.code).to.equal(dagPB.code) - expect(cid.toString(base58btc)).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr') + expect(cid.toV0().toString()).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr') const result = await ipfs.dag.get(cid) expect(result.value.Data).to.deep.equal(data) }) - it('should be able to put and get a DAG node with format dag-cbor', async () => { + it('should be able to put and get a DAG node with dag-cbor codec', async () => { const cbor = { foo: 'dag-cbor-bar' } - const cid = await ipfs.dag.put(cbor, { format: 'dag-cbor', hashAlg: 'sha2-256' }) + const cid = await ipfs.dag.put(cbor, { storeCodec: 'dag-cbor', hashAlg: 'sha2-256' }) expect(cid.code).to.equal(dagCBOR.code) expect(cid.toString(base32)).to.equal('bafyreic6f672hnponukaacmk2mmt7vs324zkagvu4hcww6yba6kby25zce') @@ -50,9 +49,9 @@ describe('.dag', function () { expect(result.value).to.deep.equal(cbor) }) - it('should be able to put and get a DAG node with format raw', async () => { + it('should be able to put and get a DAG node with raw codec', async () => { const node = uint8ArrayFromString('some data') - const cid = await ipfs.dag.put(node, { format: 'raw', hashAlg: 'sha2-256' }) + const cid = await ipfs.dag.put(node, { storeCodec: 'raw', hashAlg: 'sha2-256' }) expect(cid.code).to.equal(raw.code) expect(cid.toString(base32)).to.equal('bafkreiata6mq425fzikf5m26temcvg7mizjrxrkn35swuybmpah2ajan5y') @@ -70,19 +69,28 @@ describe('.dag', function () { await expect(ipfs.dag.get(cid)).to.eventually.be.rejectedWith(/No codec found/) }) - it('should error when putting node with esoteric format', () => { + it('should error when putting node with esoteric codec', () => { const node = uint8ArrayFromString('some data') - return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/No codec found/) + return expect(ipfs.dag.put(node, { storeCodec: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/No codec found/) }) - it('should attempt to load an unsupported format', async () => { - let askedToLoadFormat + it('should pass through raw bytes with inputCodec', async () => { + const node = uint8ArrayFromString('blob 9\0some data') + // we don't support git-raw in the HTTP client, but inputCodec and a Uint8Array should make + // the raw data pass through to go-ipfs, which does talk git-raw + const cid = await ipfs.dag.put(node, { inputCodec: 'git-raw', storeCodec: 'git-raw', hashAlg: 'sha1' }) + expect(cid.code).to.equal(0x78) + expect(cid.toString(base32)).to.equal('baf4bcfd4azdl7vj4d4hnix75qfld6mabo4l4uwa') + }) + + it('should attempt to load an unsupported codec', async () => { + let askedToLoadCodec const ipfs2 = httpClient({ url: `http://${ipfs.apiHost}:${ipfs.apiPort}`, ipld: { - loadCodec: (format) => { - askedToLoadFormat = format === 'git-raw' + loadCodec: (codec) => { + askedToLoadCodec = codec === 'boop' return { encode: (buf) => buf } @@ -93,9 +101,9 @@ describe('.dag', function () { const node = uint8ArrayFromString('some data') // error is from go-ipfs, this means the client serialized it ok - await expect(ipfs2.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/no parser for format "git-raw"/) + await expect(ipfs2.dag.put(node, { storeCodec: 'boop', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/unknown multicodec: "boop"/) - expect(askedToLoadFormat).to.be.true() + expect(askedToLoadCodec).to.be.true() }) it('should allow formats to be specified without overwriting others', async () => { @@ -115,7 +123,7 @@ describe('.dag', function () { hello: 'world' } const cid1 = await ipfs2.dag.put(dagCborNode, { - format: 'dag-cbor', + storeCodec: 'dag-cbor', hashAlg: 'sha2-256' }) @@ -124,7 +132,7 @@ describe('.dag', function () { Links: [] } const cid2 = await ipfs2.dag.put(dagPbNode, { - format: 'dag-pb', + storeCodec: 'dag-pb', hashAlg: 'sha2-256' })