feat!: match [email protected] dag put options and semantics

Fixes: #3914
Ref: https://github.com/ipfs/go-ipfs/blob/master/CHANGELOG.md#v0100-2021-09-30

--format and --input-enc have been replaced with --input-codec and
--store-codec and mean something a little different. You now supply
raw input and instruct the server which --input-codec that data is
in which it will decode, then re-encode with --store-codec before
storing it and providing you with the CID.

We accept plain JavaScript objects to encode with --store-codec via
the API here, defaulting to dag-cbor, and send that to the server as
encoded bytes using that codec, to be stored using that codec.

If you supply an --input-codec then we assume you're supplying raw,
encoded bytes using that codec and we pass that directly on to the
server to handle.

packages/ipfs-core-types/src/dag/index.ts

@@ -107,6 +107,11 @@ export interface API<OptionExtension = {}> {
 }
 
 export interface GetOptions extends AbortOptions, PreloadOptions {
+  /**
+   * The codec that the node will be encoded in (defaults to 'dag-json')
+   */
+   inputCodec?: string
+
   /**
    * An optional path within the DAG to resolve
    */
@@ -132,9 +137,14 @@ export interface GetResult {
 
 export interface PutOptions extends AbortOptions, PreloadOptions {
   /**
-   * The codec to use to create the CID (defaults to 'dag-cbor')
+   * The codec that the input object is encoded in (defaults to 'dag-json')
+   */
+  inputCodec?: string
+
+  /**
+   * The codec that the stored object will be encoded with (defaults to 'dag-cbor')
    */
-  format?: string
+  storeCodec?: string
 
   /**
    * Multihash hashing algorithm to use (defaults to 'sha2-256')

packages/ipfs-core/package.json

@@ -136,7 +136,7 @@
     "@types/rimraf": "^3.0.1",
     "aegir": "^35.0.3",
     "delay": "^5.0.0",
-    "go-ipfs": "0.9.1",
+    "go-ipfs": "0.10.0",
     "interface-blockstore-tests": "^1.0.0",
     "interface-ipfs-core": "^0.150.2",
     "ipfsd-ctl": "^10.0.3",

packages/ipfs-core/src/components/dag/put.js

@@ -18,23 +18,32 @@ module.exports = ({ repo, codecs, hashers, preload }) => {
     const release = options.pin ? await repo.gcLock.readLock() : null
 
     try {
-      const codecName = options.format || 'dag-cbor'
-      const cidVersion = options.version != null ? options.version : (codecName === 'dag-pb' ? 0 : 1)
-      const codec = await codecs.getCodec(codecName)
+      const storeCodec = await codecs.getCodec(options.storeCodec || 'dag-cbor')
+      if (!storeCodec) {
+        throw new Error(`Unknown storeCodec ${options.storeCodec}, please configure additional BlockCodecs for this IPFS instance`)
+      }
 
-      if (!codec) {
-        throw new Error(`Unknown codec ${options.format}, please configure additional BlockCodecs for this IPFS instance`)
+      if (options.inputCodec) {
+        if (!(dagNode instanceof Uint8Array)) {
+          throw new Error('Can only inputCodec on raw bytes that can be decoded')
+        }
+        const inputCodec = await codecs.getCodec(options.inputCodec)
+        if (!inputCodec) {
+          throw new Error(`Unknown inputCodec ${options.inputCodec}, please configure additional BlockCodecs for this IPFS instance`)
+        }
+        dagNode = inputCodec.decode(dagNode)
       }
 
+      const cidVersion = options.version != null ? options.version : (options.storeCodec === 'dag-pb' ? 0 : 1)
       const hasher = await hashers.getHasher(options.hashAlg || 'sha2-256')
 
       if (!hasher) {
         throw new Error(`Unknown hash algorithm ${options.hashAlg}, please configure additional MultihashHashers for this IPFS instance`)
       }
 
-      const buf = codec.encode(dagNode)
+      const buf = storeCodec.encode(dagNode)
       const hash = await hasher.digest(buf)
-      const cid = CID.create(cidVersion, codec.code, hash)
+      const cid = CID.create(cidVersion, storeCodec.code, hash)
 
       await repo.blocks.put(cid, buf, {
         signal: options.signal

packages/ipfs-http-client/package.json

@@ -69,7 +69,7 @@
   "devDependencies": {
     "aegir": "^35.0.3",
     "delay": "^5.0.0",
-    "go-ipfs": "0.9.1",
+    "go-ipfs": "0.10.0",
     "ipfsd-ctl": "^10.0.3",
     "it-all": "^1.0.4",
     "it-first": "^1.0.4",

packages/ipfs-http-client/src/dag/put.js

@@ -23,14 +23,31 @@ module.exports = (codecs, options) => {
      */
     const put = async (dagNode, options = {}) => {
       const settings = {
-        format: 'dag-cbor',
+        storeCodec: 'dag-cbor',
         hashAlg: 'sha2-256',
-        inputEnc: 'raw',
         ...options
       }
 
-      const codec = await codecs.getCodec(settings.format)
-      const serialized = codec.encode(dagNode)
+      let serialized
+
+      if (settings.inputCodec) {
+        // if you supply an inputCodec, we assume you're passing in a raw, encoded
+        // block using that codec, so we'll just pass that on to the server and let
+        // it deal with the decode/encode/store cycle
+        if (!(dagNode instanceof Uint8Array)) {
+          throw new Error('Can only inputCodec on raw bytes that can be decoded')
+        }
+        serialized = dagNode
+      } else {
+        // if you don't supply an inputCodec, we assume you've passed in a JavaScript
+        // object you want to have encoded using storeCodec, so we'll prepare it for
+        // you if we have the codec
+        const storeCodec = await codecs.getCodec(settings.storeCodec)
+        serialized = storeCodec.encode(dagNode)
+        // now we have a serialized form, the server should be told to receive it
+        // in that format
+        settings.inputCodec = settings.storeCodec
+      }
 
       // allow aborting requests on body errors
       const controller = new AbortController()

packages/ipfs-http-client/test/dag.spec.js

@@ -5,6 +5,7 @@
 
 const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
 const { expect } = require('aegir/utils/chai')
+const { CID } = require('multiformats/cid')
 const dagPb = require('@ipld/dag-pb')
 const dagCbor = require('@ipld/dag-cbor')
 const raw = require('multiformats/codecs/raw')
@@ -23,25 +24,25 @@ describe('.dag', function () {
 
   after(() => f.clean())
 
-  it('should be able to put and get a DAG node with format dag-pb', async () => {
+  it('should be able to put and get a DAG node with dag-pb codec', async () => {
     const data = uint8ArrayFromString('some data')
     const node = {
       Data: data,
       Links: []
     }
 
-    const cid = await ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256', cidVersion: 0 })
+    const cid = await ipfs.dag.put(node, { storeCodec: 'dag-pb', hashAlg: 'sha2-256' })
     expect(cid.code).to.equal(dagPb.code)
-    expect(cid.toString(base58btc)).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr')
+    expect(cid.toV0().toString()).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr')
 
     const result = await ipfs.dag.get(cid)
 
     expect(result.value.Data).to.deep.equal(data)
   })
 
-  it('should be able to put and get a DAG node with format dag-cbor', async () => {
+  it('should be able to put and get a DAG node with dag-cbor codec', async () => {
     const cbor = { foo: 'dag-cbor-bar' }
-    const cid = await ipfs.dag.put(cbor, { format: 'dag-cbor', hashAlg: 'sha2-256' })
+    const cid = await ipfs.dag.put(cbor, { storeCodec: 'dag-cbor', hashAlg: 'sha2-256' })
 
     expect(cid.code).to.equal(dagCbor.code)
     expect(cid.toString(base32)).to.equal('bafyreic6f672hnponukaacmk2mmt7vs324zkagvu4hcww6yba6kby25zce')
@@ -51,9 +52,9 @@ describe('.dag', function () {
     expect(result.value).to.deep.equal(cbor)
   })
 
-  it('should be able to put and get a DAG node with format raw', async () => {
+  it('should be able to put and get a DAG node with raw codec', async () => {
     const node = uint8ArrayFromString('some data')
-    const cid = await ipfs.dag.put(node, { format: 'raw', hashAlg: 'sha2-256' })
+    const cid = await ipfs.dag.put(node, { storeCodec: 'raw', hashAlg: 'sha2-256' })
 
     expect(cid.code).to.equal(raw.code)
     expect(cid.toString(base32)).to.equal('bafkreiata6mq425fzikf5m26temcvg7mizjrxrkn35swuybmpah2ajan5y')
@@ -71,19 +72,28 @@ describe('.dag', function () {
     await expect(ipfs.dag.get(cid)).to.eventually.be.rejectedWith(/No codec found/)
   })
 
-  it('should error when putting node with esoteric format', () => {
+  it('should error when putting node with esoteric codec', () => {
     const node = uint8ArrayFromString('some data')
 
-    return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/No codec found/)
+    return expect(ipfs.dag.put(node, { storeCodec: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/No codec found/)
   })
 
-  it('should attempt to load an unsupported format', async () => {
-    let askedToLoadFormat
+  it('should attempt to load an unsupported codec', async () => {
+    const node = uint8ArrayFromString('blob 9\0some data')
+    // we don't support git-raw in the HTTP client, but inputCodec and a Uint8Array should make
+    // the raw data pass through to go-ipfs, which does talk git-raw
+    const cid = await ipfs.dag.put(node, { inputCodec: 'git-raw', storeCodec: 'git-raw', hashAlg: 'sha1' })
+    expect(cid.code).to.equal(0x78)
+    expect(cid.toString(base32)).to.equal('baf4bcfd4azdl7vj4d4hnix75qfld6mabo4l4uwa')
+  })
+
+  it('should attempt to load an unsupported codec', async () => {
+    let askedToLoadCodec
     const ipfs2 = ipfsHttpClient.create({
       url: `http://${ipfs.apiHost}:${ipfs.apiPort}`,
       ipld: {
-        loadCodec: (format) => {
-          askedToLoadFormat = format === 'git-raw'
+        loadCodec: (codec) => {
+          askedToLoadCodec = codec === 'boop'
           return {
             encode: (buf) => buf
           }
@@ -94,12 +104,12 @@ describe('.dag', function () {
     const node = uint8ArrayFromString('some data')
 
     // error is from go-ipfs, this means the client serialized it ok
-    await expect(ipfs2.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/no parser for format "git-raw"/)
+    await expect(ipfs2.dag.put(node, { storeCodec: 'boop', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/unknown multicodec: "boop"/)
 
-    expect(askedToLoadFormat).to.be.true()
+    expect(askedToLoadCodec).to.be.true()
   })
 
-  it('should allow formats to be specified without overwriting others', async () => {
+  it('should allow codecs to be specified without overwriting others', async () => {
     const ipfs2 = ipfsHttpClient.create({
       url: `http://${ipfs.apiHost}:${ipfs.apiPort}`,
       ipld: {
@@ -116,7 +126,7 @@ describe('.dag', function () {
       hello: 'world'
     }
     const cid1 = await ipfs2.dag.put(dagCborNode, {
-      format: 'dag-cbor',
+      storeCodec: 'dag-cbor',
       hashAlg: 'sha2-256'
     })
 
@@ -125,7 +135,7 @@ describe('.dag', function () {
       Links: []
     }
     const cid2 = await ipfs2.dag.put(dagPbNode, {
-      format: 'dag-pb',
+      storeCodec: 'dag-pb',
       hashAlg: 'sha2-256'
     })
 

packages/ipfs-http-server/src/api/resources/dag.js

@@ -129,7 +129,8 @@ exports.put = {
           throw Boom.badRequest("File argument 'object data' is required")
         }
 
-        const enc = request.query.inputEncoding
+        const inputCodec = request.query.inputCodec
+        const storeCodec = request.query.storeCodec
 
         if (!request.headers['content-type']) {
           throw Boom.badRequest("File argument 'object data' is required")
@@ -149,42 +150,19 @@ exports.put = {
           throw Boom.badRequest("File argument 'object data' is required")
         }
 
-        let format = request.query.format
+        const cidVersion = format === 'dag-pb' && request.query.hashAlg === 'sha2-256' ? request.query.version : 1
 
-        if (format === 'cbor') {
-          format = 'dag-cbor'
-        }
-
-        let node
+        const cid = await request.server.app.ipfs.block.put(data, {
+          version: cidVersion,
+          storeCodec,
+          inputCodec,
+          mhtype: request.query.hash
+        })
 
-        if (format === 'raw') {
-          node = data
-        } else if (enc === 'json') {
-          try {
-            node = JSON.parse(data.toString())
-          } catch (err) {
-            throw Boom.badRequest('Failed to parse the JSON: ' + err)
-          }
-        } else {
-          // the node is an uncommon format which the client should have
-          // serialized so add it to the block store and fetch it deserialized
-          // before continuing
-          const cidVersion = format === 'dag-pb' && request.query.hashAlg === 'sha2-256' ? request.query.version : 1
-
-          const cid = await request.server.app.ipfs.block.put(data, {
-            version: cidVersion,
-            format,
-            mhtype: request.query.hash
-          })
-
-          const {
-            value
-          } = await request.server.app.ipfs.dag.get(cid)
-          node = value
-        }
+        const { value } = await request.server.app.ipfs.dag.get(cid)
 
         return {
-          node,
+          node: value,
           format,
           hashAlg: request.query.hash
         }
@@ -196,18 +174,14 @@ exports.put = {
         stripUnknown: true
       },
       query: Joi.object().keys({
-        format: Joi.string().default('cbor'),
-        inputEncoding: Joi.string().default('json'),
+        storeCodec: Joi.string().default('dag-cbor'),
+        inputCodec: Joi.string().default('dag-json'),
         pin: Joi.boolean().default(false),
         hash: Joi.string().default('sha2-256'),
         cidBase: Joi.string().default('base32'),
         version: Joi.number().integer().valid(0, 1).default(1),
         timeout: Joi.timeout()
       })
-        .rename('input-enc', 'inputEncoding', {
-          override: true,
-          ignoreUndefined: true
-        })
         .rename('cid-base', 'cidBase', {
           override: true,
           ignoreUndefined: true

packages/ipfs/package.json

@@ -51,7 +51,7 @@
     "assert": "^2.0.0",
     "cross-env": "^7.0.0",
     "electron-webrtc": "^0.3.0",
-    "go-ipfs": "0.9.1",
+    "go-ipfs": "0.10.0",
     "interface-ipfs-core": "^0.150.2",
     "ipfs-client": "^0.6.4",
     "ipfs-core-types": "^0.7.1",