feat!: upload/list items have a shards array property (#229)

- update tests and types to handle upload/list items with a shards array property. - update upload/add fn to return an UploadAddResponse as shards out may be more than shards in. - update README see also: storacha/w3infra#56 Signed-off-by: Oli Evans <[email protected]>
storacha · Dec 5, 2022 · 723b281 · 723b281
1 parent 197439e
commit 723b281
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 74 deletions.
diff --git a/packages/upload-client/README.md b/packages/upload-client/README.md
@@ -347,7 +347,7 @@ function add(
   root: CID,
   shards: CID[],
   options: { retries?: number; signal?: AbortSignal } = {}
-): Promise<void>
+): Promise<UploadAddResponse>
 ```
 
 Register a set of stored CAR files as an "upload" in the system. A DAG can be split between multipe CAR files. Calling this function allows multiple stored CAR files to be considered as a single upload.

diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts
@@ -34,7 +34,7 @@ export interface Service {
     remove: ServiceMethod<StoreRemove, null, never>
   }
   upload: {
-    add: ServiceMethod<UploadAdd, null, never>
+    add: ServiceMethod<UploadAdd, UploadAddResponse, never>
     list: ServiceMethod<UploadList, ListResponse<UploadListResult>, never>
     remove: ServiceMethod<UploadRemove, null, never>
   }
@@ -46,25 +46,24 @@ export interface StoreAddResponse {
   url: string
 }
 
+export interface UploadAddResponse {
+  root: AnyLink
+  shards?: CARLink[]
+}
+
 export interface ListResponse<R> {
   cursor?: string
   size: number
   results: R[]
 }
 
 export interface StoreListResult {
-  payloadCID: string
-  origin?: string
+  link: CARLink
   size: number
-  uploadedAt: string
+  origin?: CARLink
 }
 
-export interface UploadListResult {
-  uploaderDID: string
-  dataCID: string
-  carCID: string
-  uploadedAt: string
-}
+export interface UploadListResult extends UploadAddResponse {}
 
 export interface InvocationConfig {
   /**

diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js
@@ -25,6 +25,7 @@ import { REQUEST_RETRIES } from './constants.js'
  * @param {import('multiformats/link').UnknownLink} root Root data CID for the DAG that was stored.
  * @param {import('./types').CARLink[]} shards CIDs of CAR files that contain the DAG.
  * @param {import('./types').RequestOptions} [options]
+ * @returns {Promise<import('./types').UploadAddResponse>}
  */
 export async function add(
   { issuer, with: resource, proofs, audience = servicePrincipal },
@@ -57,6 +58,8 @@ export async function add(
       cause: result,
     })
   }
+
+  return result
 }
 
 /**

diff --git a/packages/upload-client/test/helpers/car.js b/packages/upload-client/test/helpers/car.js
@@ -0,0 +1,26 @@
+import { CarWriter } from '@ipld/car'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as CAR from '@ucanto/transport/car'
+
+/**
+ * @param {Uint8Array} bytes
+ **/
+export async function toCAR(bytes) {
+  const hash = await sha256.digest(bytes)
+  const root = CID.create(1, raw.code, hash)
+
+  const { writer, out } = CarWriter.create(root)
+  writer.put({ cid: root, bytes })
+  writer.close()
+
+  const chunks = []
+  for await (const chunk of out) {
+    chunks.push(chunk)
+  }
+  const blob = new Blob(chunks)
+  const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer()))
+
+  return Object.assign(blob, { cid, roots: [root] })
+}
diff --git a/packages/upload-client/test/helpers/random.js b/packages/upload-client/test/helpers/random.js
@@ -1,8 +1,4 @@
-import { CarWriter } from '@ipld/car'
-import { CID } from 'multiformats/cid'
-import * as raw from 'multiformats/codecs/raw'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as CAR from '@ucanto/transport/car'
+import { toCAR } from './car.js'
 
 /** @param {number} size */
 export async function randomBytes(size) {
@@ -31,19 +27,5 @@ export async function randomBytes(size) {
 /** @param {number} size */
 export async function randomCAR(size) {
   const bytes = await randomBytes(size)
-  const hash = await sha256.digest(bytes)
-  const root = CID.create(1, raw.code, hash)
-
-  const { writer, out } = CarWriter.create(root)
-  writer.put({ cid: root, bytes })
-  writer.close()
-
-  const chunks = []
-  for await (const chunk of out) {
-    chunks.push(chunk)
-  }
-  const blob = new Blob(chunks)
-  const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer()))
-
-  return Object.assign(blob, { cid, roots: [root] })
+  return toCAR(bytes)
 }
diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js
@@ -10,6 +10,7 @@ import * as UploadCapabilities from '@web3-storage/capabilities/upload'
 import { uploadFile, uploadDirectory } from '../src/index.js'
 import { serviceSigner } from './fixtures.js'
 import { randomBytes } from './helpers/random.js'
+import { toCAR } from './helpers/car.js'
 import { File } from './helpers/shims.js'
 import { mockService } from './helpers/mocks.js'
 
@@ -23,7 +24,10 @@ describe('uploadFile', () => {
 
     const space = await Signer.generate()
     const agent = await Signer.generate() // The "user" that will ask the service to accept the upload
-    const file = new Blob([await randomBytes(128)])
+    const bytes = await randomBytes(128)
+    const file = new Blob([bytes])
+    const expectedCar = await toCAR(bytes)
+
     /** @type {import('../src/types').CARLink|undefined} */
     let carCID
 
@@ -62,7 +66,10 @@ describe('uploadFile', () => {
           assert.equal(invCap.with, space.did())
           assert.equal(invCap.nb?.shards?.length, 1)
           assert.equal(String(invCap.nb?.shards?.[0]), carCID?.toString())
-          return null
+          return {
+            root: expectedCar.roots[0],
+            shards: [expectedCar.cid],
+          }
         }),
       },
     })
@@ -95,8 +102,8 @@ describe('uploadFile', () => {
     assert(service.upload.add.called)
     assert.equal(service.upload.add.callCount, 1)
 
-    assert(carCID)
-    assert(dataCID)
+    assert.equal(carCID?.toString(), expectedCar.cid.toString())
+    assert.equal(dataCID.toString(), expectedCar.roots[0].toString())
   })
 
   it('allows custom shard size to be set', async () => {
@@ -129,7 +136,12 @@ describe('uploadFile', () => {
 
     const service = mockService({
       store: { add: provide(StoreCapabilities.add, () => res) },
-      upload: { add: provide(UploadCapabilities.add, () => null) },
+      upload: {
+        add: provide(UploadCapabilities.add, ({ capability }) => {
+          if (!capability.nb) throw new Error('nb must be present')
+          return capability.nb
+        }),
+      },
     })
 
     const server = Server.create({
@@ -210,7 +222,8 @@ describe('uploadDirectory', () => {
           assert.equal(invCap.with, space.did())
           assert.equal(invCap.nb?.shards?.length, 1)
           assert.equal(String(invCap.nb?.shards?.[0]), carCID?.toString())
-          return null
+          if (!invCap.nb) throw new Error('nb must be present')
+          return invCap.nb
         }),
       },
     })
@@ -277,7 +290,12 @@ describe('uploadDirectory', () => {
 
     const service = mockService({
       store: { add: provide(StoreCapabilities.add, () => res) },
-      upload: { add: provide(UploadCapabilities.add, () => null) },
+      upload: {
+        add: provide(UploadCapabilities.add, ({ capability }) => {
+          if (!capability.nb) throw new Error('nb must be present')
+          return capability.nb
+        }),
+      },
     })
 
     const server = Server.create({

diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js
@@ -294,9 +294,8 @@ describe('Store.list', () => {
       size: 1000,
       results: [
         {
-          payloadCID: car.cid.toString(),
+          link: car.cid,
           size: 123,
-          uploadedAt: new Date().toISOString(),
         },
       ],
     }
@@ -352,9 +351,8 @@ describe('Store.list', () => {
     assert(list.results)
     assert.equal(list.results.length, res.results.length)
     list.results.forEach((r, i) => {
-      assert.equal(r.payloadCID, res.results[i].payloadCID)
-      assert.equal(r.size, res.results[i].size)
-      assert.equal(r.uploadedAt, res.results[i].uploadedAt)
+      assert.deepEqual(r.link, res.results[i].link)
+      assert.deepEqual(r.size, res.results[i].size)
     })
   })
 
@@ -365,19 +363,17 @@ describe('Store.list', () => {
       size: 1,
       results: [
         {
-          payloadCID: (await randomCAR(128)).cid.toString(),
+          link: (await randomCAR(128)).cid,
           size: 123,
-          uploadedAt: new Date().toISOString(),
         },
       ],
     }
     const page1 = {
       size: 1,
       results: [
         {
-          payloadCID: (await randomCAR(128)).cid.toString(),
+          link: (await randomCAR(128)).cid,
           size: 123,
-          uploadedAt: new Date().toISOString(),
         },
       ],
     }
@@ -437,18 +433,16 @@ describe('Store.list', () => {
     assert(results0.results)
     assert.equal(results0.results.length, page0.results.length)
     results0.results.forEach((r, i) => {
-      assert.equal(r.payloadCID, page0.results[i].payloadCID)
+      assert.equal(r.link.toString(), page0.results[i].link.toString())
       assert.equal(r.size, page0.results[i].size)
-      assert.equal(r.uploadedAt, page0.results[i].uploadedAt)
     })
 
     assert(results1.results)
     assert.equal(results1.cursor, undefined)
     assert.equal(results1.results.length, page1.results.length)
     results1.results.forEach((r, i) => {
-      assert.equal(r.payloadCID, page1.results[i].payloadCID)
+      assert.equal(r.link.toString(), page1.results[i].link.toString())
       assert.equal(r.size, page1.results[i].size)
-      assert.equal(r.uploadedAt, page1.results[i].uploadedAt)
     })
   })
 

diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js
@@ -17,6 +17,11 @@ describe('Upload.add', () => {
     const agent = await Signer.generate()
     const car = await randomCAR(128)
 
+    const res = {
+      root: car.roots[0],
+      shards: [car.cid],
+    }
+
     const proofs = [
       await UploadCapabilities.add.delegate({
         issuer: space,
@@ -37,7 +42,7 @@ describe('Upload.add', () => {
           assert.equal(String(invCap.nb?.root), car.roots[0].toString())
           assert.equal(invCap.nb?.shards?.length, 1)
           assert.equal(String(invCap.nb?.shards?.[0]), car.cid.toString())
-          return null
+          return res
         }),
       },
     })
@@ -56,7 +61,7 @@ describe('Upload.add', () => {
     })
 
     const root = car.roots[0]
-    await Upload.add(
+    const actual = await Upload.add(
       { issuer: agent, with: space.did(), proofs, audience: serviceSigner },
       root,
       [car.cid],
@@ -65,6 +70,11 @@ describe('Upload.add', () => {
 
     assert(service.upload.add.called)
     assert.equal(service.upload.add.callCount, 1)
+    assert.equal(actual.root.toString(), res.root.toString())
+    assert.deepEqual(
+      new Set(actual.shards?.map((s) => s.toString())),
+      new Set(res.shards.map((s) => s.toString()))
+    )
   })
 
   it('throws on service error', async () => {
@@ -127,10 +137,8 @@ describe('Upload.list', () => {
       size: 1000,
       results: [
         {
-          uploaderDID: agent.did(),
-          carCID: car.cid.toString(),
-          dataCID: car.roots[0].toString(),
-          uploadedAt: new Date().toISOString(),
+          root: car.roots[0],
+          shards: [car.cid],
         },
       ],
     }
@@ -182,9 +190,11 @@ describe('Upload.list', () => {
     assert(list.results)
     assert.equal(list.results.length, res.results.length)
     list.results.forEach((r, i) => {
-      assert.equal(r.carCID.toString(), res.results[i].carCID.toString())
-      assert.equal(r.dataCID.toString(), res.results[i].dataCID.toString())
-      assert.equal(r.uploadedAt, res.results[i].uploadedAt)
+      assert.equal(r.root.toString(), res.results[i].root.toString())
+      assert.deepStrictEqual(
+        new Set(r.shards?.map((s) => s.toString())),
+        new Set(res.results[i].shards.map((s) => s.toString()))
+      )
     })
   })
 
@@ -199,10 +209,8 @@ describe('Upload.list', () => {
       size: 1,
       results: [
         {
-          uploaderDID: agent.did(),
-          carCID: car0.cid.toString(),
-          dataCID: car0.roots[0].toString(),
-          uploadedAt: new Date().toISOString(),
+          root: car0.roots[0],
+          shards: [car0.cid],
         },
       ],
     }
@@ -211,10 +219,8 @@ describe('Upload.list', () => {
       size: 1,
       results: [
         {
-          uploaderDID: agent.did(),
-          carCID: car1.cid.toString(),
-          dataCID: car1.roots[0].toString(),
-          uploadedAt: new Date().toISOString(),
+          root: car1.roots[0],
+          shards: [car1.cid],
         },
       ],
     }
@@ -271,19 +277,23 @@ describe('Upload.list', () => {
     assert(results0.results)
     assert.equal(results0.results.length, page0.results.length)
     results0.results.forEach((r, i) => {
-      assert.equal(r.carCID.toString(), page0.results[i].carCID.toString())
-      assert.equal(r.dataCID.toString(), page0.results[i].dataCID.toString())
-      assert.equal(r.uploadedAt, page0.results[i].uploadedAt)
+      assert.equal(r.root.toString(), page0.results[i].root.toString())
+      assert.deepStrictEqual(
+        new Set(r.shards?.map((s) => s.toString())),
+        new Set(page0.results[i].shards.map((s) => s.toString()))
+      )
     })
 
     assert.equal(results1.cursor, undefined)
     assert.equal(results1.size, page1.size)
     assert(results1.results)
     assert.equal(results1.results.length, page1.results.length)
     results1.results.forEach((r, i) => {
-      assert.equal(r.carCID.toString(), page1.results[i].carCID.toString())
-      assert.equal(r.dataCID.toString(), page1.results[i].dataCID.toString())
-      assert.equal(r.uploadedAt, page1.results[i].uploadedAt)
+      assert.equal(r.root.toString(), page1.results[i].root.toString())
+      assert.deepStrictEqual(
+        new Set(r.shards?.map((s) => s.toString())),
+        new Set(page1.results[i].shards.map((s) => s.toString()))
+      )
     })
   })