feat!: add blob list and remove (#1385)

Implements storacha/specs#123

BREAKING CHANGE: allocations storage interface now requires remove to be
implemented

packages/capabilities/src/blob.js

@@ -11,7 +11,8 @@
  *
  * @module
  */
-import { capability, Schema } from '@ucanto/validator'
+import { equals } from 'uint8arrays/equals'
+import { capability, Schema, fail, ok } from '@ucanto/validator'
 import { equalBlob, equalWith, SpaceDID } from './utils.js'
 
 /**
@@ -70,6 +71,72 @@ export const add = capability({
   derives: equalBlob,
 })
 
+/**
+ * Capability can be used to remove the stored Blob from the (memory)
+ * space identified by `with` field.
+ */
+export const remove = capability({
+  can: 'blob/remove',
+  /**
+   * DID of the (memory) space where Blob is stored.
+   */
+  with: SpaceDID,
+  nb: Schema.struct({
+    /**
+     * A multihash digest of the blob payload bytes, uniquely identifying blob.
+     */
+    digest: Schema.bytes(),
+  }),
+  derives: (claimed, delegated) => {
+    if (claimed.with !== delegated.with) {
+      return fail(
+        `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'`
+      )
+    } else if (
+      delegated.nb.digest &&
+      !equals(delegated.nb.digest, claimed.nb.digest)
+    ) {
+      return fail(
+        `Link ${
+          claimed.nb.digest ? `${claimed.nb.digest}` : ''
+        } violates imposed ${delegated.nb.digest} constraint.`
+      )
+    }
+    return ok({})
+  },
+})
+
+/**
+ * Capability can be invoked to request a list of stored Blobs in the
+ * (memory) space identified by `with` field.
+ */
+export const list = capability({
+  can: 'blob/list',
+  /**
+   * DID of the (memory) space where Blobs to be listed are stored.
+   */
+  with: SpaceDID,
+  nb: Schema.struct({
+    /**
+     * A pointer that can be moved back and forth on the list.
+     * It can be used to paginate a list for instance.
+     */
+    cursor: Schema.string().optional(),
+    /**
+     * Maximum number of items per page.
+     */
+    size: Schema.integer().optional(),
+  }),
+  derives: (claimed, delegated) => {
+    if (claimed.with !== delegated.with) {
+      return fail(
+        `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'`
+      )
+    }
+    return ok({})
+  },
+})
+
 // ⚠️ We export imports here so they are not omitted in generated typedefs
 // @see https://github.com/microsoft/TypeScript/issues/51548
 export { Schema }

packages/capabilities/src/index.js

@@ -95,6 +95,8 @@ export const abilitiesAsStrings = [
   Usage.report.can,
   Blob.blob.can,
   Blob.add.can,
+  Blob.remove.can,
+  Blob.list.can,
   W3sBlob.blob.can,
   W3sBlob.allocate.can,
   W3sBlob.accept.can,

packages/capabilities/src/types.ts

@@ -452,6 +452,8 @@ export type HTTPPut = InferInvokedCapability<typeof HTTPCaps.put>
 // Blob
 export type Blob = InferInvokedCapability<typeof BlobCaps.blob>
 export type BlobAdd = InferInvokedCapability<typeof BlobCaps.add>
+export type BlobRemove = InferInvokedCapability<typeof BlobCaps.remove>
+export type BlobList = InferInvokedCapability<typeof BlobCaps.list>
 export type ServiceBlob = InferInvokedCapability<typeof W3sBlobCaps.blob>
 export type BlobAllocate = InferInvokedCapability<typeof W3sBlobCaps.allocate>
 export type BlobAccept = InferInvokedCapability<typeof W3sBlobCaps.accept>
@@ -487,6 +489,20 @@ export interface BlobListItem {
   insertedAt: ISO8601Date
 }
 
+// Blob remove
+export interface BlobRemoveSuccess {
+  size: number
+}
+
+// TODO: make types more specific
+export type BlobRemoveFailure = Ucanto.Failure
+
+// Blob list
+export interface BlobListSuccess extends ListResponse<BlobListItem> {}
+
+// TODO: make types more specific
+export type BlobListFailure = Ucanto.Failure
+
 // Blob allocate
 export interface BlobAllocateSuccess {
   size: number
@@ -820,6 +836,8 @@ export type ServiceAbilityArray = [
   UsageReport['can'],
   Blob['can'],
   BlobAdd['can'],
+  BlobRemove['can'],
+  BlobList['can'],
   ServiceBlob['can'],
   BlobAllocate['can'],
   BlobAccept['can'],

packages/upload-api/src/blob.js

@@ -1,4 +1,6 @@
 import { blobAddProvider } from './blob/add.js'
+import { blobListProvider } from './blob/list.js'
+import { blobRemoveProvider } from './blob/remove.js'
 import * as API from './types.js'
 
 /**
@@ -7,5 +9,7 @@ import * as API from './types.js'
 export function createService(context) {
   return {
     add: blobAddProvider(context),
+    list: blobListProvider(context),
+    remove: blobRemoveProvider(context),
   }
 }

packages/upload-api/src/blob/add.js

@@ -207,9 +207,7 @@ async function put({ context, blob, allocateTask }) {
   // of the `http/put` invocation. That way anyone with blob digest
   // could perform the invocation and issue receipt by deriving same
   // principal
-  const blobProvider = await ed25519.derive(
-    blob.digest.subarray(-32)
-  )
+  const blobProvider = await ed25519.derive(blob.digest.subarray(-32))
   const facts = [
     {
       keys: blobProvider.toArchive(),

packages/upload-api/src/blob/list.js

@@ -0,0 +1,15 @@
+import * as Server from '@ucanto/server'
+import * as Blob from '@web3-storage/capabilities/blob'
+import * as API from '../types.js'
+
+/**
+ * @param {API.BlobServiceContext} context
+ * @returns {API.ServiceMethod<API.BlobList, API.BlobListSuccess, API.Failure>}
+ */
+export function blobListProvider(context) {
+  return Server.provide(Blob.list, async ({ capability }) => {
+    const space = capability.with
+    const { cursor, size } = capability.nb
+    return await context.allocationsStorage.list(space, { size, cursor })
+  })
+}

packages/upload-api/src/blob/remove.js

@@ -0,0 +1,26 @@
+import * as Server from '@ucanto/server'
+import * as Blob from '@web3-storage/capabilities/blob'
+import * as API from '../types.js'
+
+import { RecordNotFoundErrorName } from '../errors.js'
+
+/**
+ * @param {API.BlobServiceContext} context
+ * @returns {API.ServiceMethod<API.BlobRemove, API.BlobRemoveSuccess, API.BlobRemoveFailure>}
+ */
+export function blobRemoveProvider(context) {
+  return Server.provide(Blob.remove, async ({ capability }) => {
+    const space = capability.with
+    const { digest } = capability.nb
+    const res = await context.allocationsStorage.remove(space, digest)
+    if (res.error && res.error.name === RecordNotFoundErrorName) {
+      return {
+        ok: {
+          size: 0,
+        },
+      }
+    }
+
+    return res
+  })
+}

packages/upload-api/src/types.ts

@@ -57,6 +57,12 @@ import {
   BlobAdd,
   BlobAddSuccess,
   BlobAddFailure,
+  BlobList,
+  BlobListSuccess,
+  BlobListFailure,
+  BlobRemove,
+  BlobRemoveSuccess,
+  BlobRemoveFailure,
   BlobAllocate,
   BlobAllocateSuccess,
   BlobAllocateFailure,
@@ -186,6 +192,8 @@ export type { AllocationsStorage, BlobsStorage, TasksStorage, BlobAddInput }
 export interface Service extends StorefrontService, W3sService {
   blob: {
     add: ServiceMethod<BlobAdd, BlobAddSuccess, BlobAddFailure>
+    remove: ServiceMethod<BlobRemove, BlobRemoveSuccess, BlobRemoveFailure>
+    list: ServiceMethod<BlobList, BlobListSuccess, BlobListFailure>
   }
   store: {
     add: ServiceMethod<StoreAdd, StoreAddSuccess, Failure>

packages/upload-api/src/types/blob.ts

@@ -5,9 +5,13 @@ import type {
   Failure,
   DID,
 } from '@ucanto/interface'
-import { BlobMultihash, BlobListItem } from '@web3-storage/capabilities/types'
+import {
+  BlobMultihash,
+  BlobListItem,
+  BlobRemoveSuccess,
+} from '@web3-storage/capabilities/types'
 
-import { RecordKeyConflict, ListOptions, ListResponse } from '../types.js'
+import { RecordKeyConflict, ListResponse } from '../types.js'
 import { Storage } from './storage.js'
 
 export type TasksStorage = Storage<UnknownLink, Invocation>
@@ -29,6 +33,16 @@ export interface AllocationsStorage {
     space: DID,
     options?: ListOptions
   ) => Promise<Result<ListResponse<BlobListItem>, Failure>>
+  /** Removes an item from the table, returning zero on size if non existent. */
+  remove: (
+    space: DID,
+    digest: BlobMultihash
+  ) => Promise<Result<BlobRemoveSuccess, Failure>>
+}
+
+export interface ListOptions {
+  size?: number
+  cursor?: string
 }
 
 export interface BlobModel {
@@ -42,8 +56,7 @@ export interface BlobAddInput {
   blob: BlobModel
 }
 
-export interface BlobAddOutput
-  extends Omit<BlobAddInput, 'space' | 'cause'> {}
+export interface BlobAddOutput extends Omit<BlobAddInput, 'space' | 'cause'> {}
 
 export interface BlobGetOutput {
   blob: { digest: Uint8Array; size: number }