diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index b41d5b857..fde33d0bb 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -1,5 +1,6 @@ import * as Blob from '@storacha/capabilities/blob' -import { Message, Invocation } from '@ucanto/core' +import * as W3sBlob from '@storacha/capabilities/web3.storage/blob' +import { Message, Receipt, Invocation } from '@ucanto/core' import * as Transport from '@ucanto/transport/car' import * as API from '../types.js' import * as HTTP from '@storacha/capabilities/http' @@ -83,10 +84,34 @@ export const poll = async (context, receipt) => { configure.ok.connection ) + // Create receipt for legacy `web3.storage/blob/accept`. The old client + // `@web3-storage/w3up-client` will poll for a receipt for this task, so + // we create one whose result is simply the result of the actual `blob/accept` + // task. + // + // TODO: remove when all users migrate to `@storacha/client`. + const w3sAccept = W3sBlob.accept.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + blob: allocate.nb.blob, + space: /** @type {API.DIDKey} */ (DID.decode(allocate.nb.space).did()), + _put: { 'ucan/await': ['.out.ok', receipt.ran.link()] }, + }, + }) + const w3sAcceptTask = await w3sAccept.delegate() + const w3sAcceptReceipt = await Receipt.issue({ + issuer: context.id, + ran: w3sAcceptTask.cid, + result: acceptReceipt.out, + fx: acceptReceipt.fx, + }) + // record the invocation and the receipt const message = await Message.build({ invocations: [configure.ok.invocation], - receipts: [acceptReceipt], + receipts: [acceptReceipt, w3sAcceptReceipt], }) const messageWrite = await context.agentStore.messages.write({ source: await Transport.outbound.encode(message), diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 5dc3a2cd4..f37372c0c 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -4,6 +4,7 @@ import * as Transport from '@ucanto/transport/car' import { ed25519 } from '@ucanto/principal' import * as Blob from '@storacha/capabilities/blob' import * as SpaceBlob from '@storacha/capabilities/space/blob' +import * as W3sBlob from '@storacha/capabilities/web3.storage/blob' import * as HTTP from '@storacha/capabilities/http' import * as Digest from 'multiformats/hashes/digest' import * as DID from '@ipld/dag-ucan/did' @@ -50,6 +51,14 @@ export function blobAddProvider(context) { return allocation } + const allocationW3s = await allocateW3s({ + context, + blob, + space, + cause: invocation.link(), + receipt: allocation.ok.receipt, + }) + const delivery = await put({ blob, allocation: allocation.ok, @@ -66,7 +75,16 @@ export function blobAddProvider(context) { return acceptance } - // Create a result describing the this invocation workflow + const acceptanceW3s = await acceptW3s({ + context, + provider: allocation.ok.provider, + blob, + space, + delivery: delivery, + acceptance: acceptance.ok, + }) + + // Create a result describing this invocation workflow let result = Server.ok({ /** @type {API.SpaceBlobAddSuccess['site']} */ site: { @@ -74,12 +92,20 @@ export function blobAddProvider(context) { }, }) .fork(allocation.ok.task) + .fork(allocationW3s.task) .fork(delivery.task) .fork(acceptance.ok.task) + .fork(acceptanceW3s.task) // As a temporary solution we fork all add effects that add inline // receipts so they can be delivered to the client. - const fx = [...allocation.ok.fx, ...delivery.fx, ...acceptance.ok.fx] + const fx = [ + ...allocation.ok.fx, + ...allocationW3s.fx, + ...delivery.fx, + ...acceptance.ok.fx, + ...acceptanceW3s.fx, + ] for (const task of fx) { result = result.fork(task) } @@ -172,6 +198,49 @@ async function allocate({ context, blob, space, cause }) { }) } +/** + * Create an allocation task and receipt using the legacy + * `web3.storage/blob/allocate` capability. This enables backwards compatibility + * with `@web3-storage/w3up-client`. + * + * TODO: remove when all users migrate to `@storacha/client`. + * + * @param {object} allocate + * @param {API.BlobServiceContext} allocate.context + * @param {API.BlobModel} allocate.blob + * @param {API.DIDKey} allocate.space + * @param {API.Link} allocate.cause + * @param {API.Receipt} allocate.receipt + */ +async function allocateW3s({ context, blob, space, cause, receipt }) { + const w3sAllocate = W3sBlob.allocate.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { blob, cause, space }, + expiration: Infinity, + }) + const w3sAllocateTask = await w3sAllocate.delegate() + + const w3sAllocateReceipt = await Receipt.issue({ + issuer: context.id, + ran: w3sAllocateTask.cid, + result: receipt.out, + }) + + const w3sAllocateConclude = createConcludeInvocation( + context.id, + context.id, + w3sAllocateReceipt + ) + + return { + task: w3sAllocateTask, + receipt: w3sAllocateReceipt, + fx: [await w3sAllocateConclude.delegate()], + } +} + /** * Create put task and check if there is a receipt for it already. * A `http/put` should be task is stored by the service, if it does not exist @@ -313,3 +382,66 @@ async function accept({ context, provider, blob, space, delivery }) { fx: receipt ? [await conclude(receipt, context.id)] : [], }) } + +/** + * Create an accept task and receipt using the legacy + * `web3.storage/blob/accept` capability. This enables backwards compatibility + * with `@web3-storage/w3up-client`. + * + * TODO: remove when all users migrate to `@storacha/client`. + * + * @param {object} input + * @param {API.BlobServiceContext} input.context + * @param {API.Principal} input.provider + * @param {API.BlobModel} input.blob + * @param {API.DIDKey} input.space + * @param {object} input.delivery + * @param {API.Invocation} input.delivery.task + * @param {API.Receipt|null} input.delivery.receipt + * @param {object} input.acceptance + * @param {API.Receipt|null} input.acceptance.receipt + */ +async function acceptW3s({ context, blob, space, delivery, acceptance }) { + // 1. Create web3.storage/blob/accept invocation and task + const w3sAccept = W3sBlob.accept.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + blob, + space, + _put: { 'ucan/await': ['.out.ok', delivery.task.link()] }, + }, + }) + const w3sAcceptTask = await w3sAccept.delegate() + + let w3sAcceptReceipt = null + // If put has failed, we propagate the error to the `blob/accept` receipt. + if (delivery.receipt?.out.error) { + w3sAcceptReceipt = await Receipt.issue({ + issuer: context.id, + ran: w3sAcceptTask, + result: { + error: new AwaitError({ + cause: delivery.receipt.out.error, + at: '.out.ok', + reference: delivery.task.link(), + }), + }, + }) + } + // If `blob/accept` receipt is present, we issue a receipt for + // `web3.storage/blob/accept`. + else if (acceptance.receipt) { + w3sAcceptReceipt = await Receipt.issue({ + issuer: context.id, + ran: w3sAcceptTask, + result: acceptance.receipt.out, + }) + } + + return { + task: w3sAcceptTask, + fx: w3sAcceptReceipt ? [await conclude(w3sAcceptReceipt, context.id)] : [], + } +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 18101f195..fba81866b 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -414,6 +414,8 @@ export interface RevocationServiceContext { } export interface ConcludeServiceContext { + /** Upload service signer. */ + id: Signer /** * Store for invocations & receipts. */ diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 3a491f49b..f0ecb1b4a 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -56,7 +56,7 @@ export const test = { blobAdd.out.ok.site['ucan/await'][1], next.accept.task.cid ) - assert.equal(blobAdd.fx.fork.length, 4) + assert.equal(blobAdd.fx.fork.length, 7) // validate receipt next assert.ok(next.allocate.task) diff --git a/packages/upload-client/src/blob/add.js b/packages/upload-client/src/blob/add.js index 0f4793767..621cbfc7e 100644 --- a/packages/upload-client/src/blob/add.js +++ b/packages/upload-client/src/blob/add.js @@ -1,5 +1,4 @@ import { ed25519 } from '@ucanto/principal' -import { conclude } from '@storacha/capabilities/ucan' import * as UCAN from '@storacha/capabilities/ucan' import { Delegation, Receipt } from '@ucanto/core' import * as BlobCapabilities from '@storacha/capabilities/blob' @@ -121,7 +120,7 @@ export function createConcludeInvocation(id, serviceDid, receipt) { receiptBlocks.push(block) receiptCids.push(block.cid) } - const concludeAllocatefx = conclude.invoke({ + const concludeAllocatefx = UCAN.conclude.invoke({ issuer: id, audience: serviceDid, with: id.toDIDKey(), @@ -292,9 +291,12 @@ export async function add( ) const ucanConclude = await httpPutConcludeInvocation.execute(conn) if (!ucanConclude.out.ok) { - throw new Error(`failed ${SpaceBlobCapabilities.add.can} invocation`, { - cause: result.out.error, - }) + throw new Error( + `failed ${UCAN.conclude.can} for ${HTTPCapabilities.put.can} invocation`, + { + cause: result.out.error, + } + ) } } diff --git a/packages/upload-client/test/blob.test.js b/packages/upload-client/test/blob.test.js index 48d66a4cc..dc9ed386a 100644 --- a/packages/upload-client/test/blob.test.js +++ b/packages/upload-client/test/blob.test.js @@ -185,7 +185,7 @@ describe('Blob.add', () => { { connection } ), { - message: 'failed space/blob/add invocation', + message: 'failed ucan/conclude for http/put invocation', } ) }) diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json index ecc5a3eb8..040224a4b 100644 --- a/packages/w3up-client/package.json +++ b/packages/w3up-client/package.json @@ -158,8 +158,10 @@ "@types/mocha": "^10.0.1", "@types/node": "^20.8.4", "@ucanto/server": "^10.0.0", + "@web3-storage/access": "^20.1.0", "@web3-storage/content-claims": "^4.0.4", "@web3-storage/data-segment": "^5.0.0", + "@web3-storage/w3up-client": "^16.5.1", "assert": "^2.0.0", "c8": "^7.13.0", "hundreds": "^0.0.9", diff --git a/packages/w3up-client/test/legacy-compat.node.test.js b/packages/w3up-client/test/legacy-compat.node.test.js new file mode 100644 index 000000000..43a4851e8 --- /dev/null +++ b/packages/w3up-client/test/legacy-compat.node.test.js @@ -0,0 +1,69 @@ +import http from 'node:http' +import { Client } from '@web3-storage/w3up-client' +import { AgentData } from '@web3-storage/access' +import * as Link from 'multiformats/link' +import { Message } from '@ucanto/core' +import * as CAR from '@ucanto/transport/car' +import * as Test from './test.js' +import { randomBytes } from './helpers/random.js' + +/** @param {import('@storacha/upload-api').AgentStore} agentStore */ +const createReceiptsServer = (agentStore) => + http.createServer(async (req, res) => { + const task = Link.parse(req.url?.split('/').pop() ?? '') + const receiptGet = await agentStore.receipts.get(task) + if (receiptGet.error) { + res.writeHead(404) + return res.end() + } + const message = await Message.build({ receipts: [receiptGet.ok] }) + const request = CAR.request.encode(message) + res.writeHead(200) + res.end(request.body) + }) + +/** @type {Test.Suite} */ +export const testLegacyCompatibility = { + uploadFile: Test.withContext({ + 'should upload a file to the service via legacy client': async ( + assert, + { connection, provisionsStorage, agentStore } + ) => { + const receiptsServer = createReceiptsServer(agentStore) + const receiptsEndpoint = await new Promise((resolve) => { + receiptsServer.listen(() => { + // @ts-expect-error + resolve(new URL(`http://127.0.0.1:${receiptsServer.address().port}`)) + }) + }) + + try { + const bytes = await randomBytes(128) + const file = new Blob([bytes]) + const alice = new Client(await AgentData.create(), { + // @ts-expect-error service no longer implements `store/*` + serviceConf: { access: connection, upload: connection }, + receiptsEndpoint, + }) + + const space = await alice.createSpace('upload-test') + const auth = await space.createAuthorization(alice) + await alice.addSpace(auth) + await alice.setCurrentSpace(space.did()) + + await provisionsStorage.put({ + // @ts-expect-error + provider: connection.id.did(), + account: alice.agent.did(), + consumer: space.did(), + }) + + await assert.doesNotReject(alice.uploadFile(file)) + } finally { + receiptsServer.close() + } + }, + }), +} + +Test.test({ LegacyCompatibility: testLegacyCompatibility }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1280c8858..7a25e3278 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -780,12 +780,18 @@ importers: '@ucanto/server': specifier: ^10.0.0 version: 10.0.0 + '@web3-storage/access': + specifier: ^20.1.0 + version: 20.1.0 '@web3-storage/content-claims': specifier: ^4.0.4 version: 4.0.5 '@web3-storage/data-segment': specifier: ^5.0.0 version: 5.3.0 + '@web3-storage/w3up-client': + specifier: ^16.5.1 + version: 16.5.1 assert: specifier: ^2.0.0 version: 2.1.0 @@ -2520,6 +2526,16 @@ packages: '@web-std/stream@1.0.0': resolution: {integrity: sha512-jyIbdVl+0ZJyKGTV0Ohb9E6UnxP+t7ZzX4Do3AHjZKxUXKMs9EmqnBDQgHF7bEw0EzbQygOjtt/7gvtmi//iCQ==} + '@web3-storage/access@20.1.0': + resolution: {integrity: sha512-IY6ICPRWE8++2jxvy+LzAiFvwAOIHR8cu9eNt+VT5sAFE796o4ma7GSU0eXRCiShmV2n6iSWAwWRT6XD5zIqPA==} + + '@web3-storage/blob-index@1.0.4': + resolution: {integrity: sha512-04+PrmVHFT+xzRhyIPdcvGc8Y2NDffUe8R1gJOyErVzEVz5N1I9Q/BrlFHYt/A4HrjM5JBsxqSrZgTIkjfPmLA==} + engines: {node: '>=16.15'} + + '@web3-storage/capabilities@17.4.1': + resolution: {integrity: sha512-GogLfON8PZabi03CUyncBvMcCi36yQ/0iR5P8kr4pxdnZm7OuAn4sEwbEB8rTKbah5V10Vwgb3O5dh0FBgyjHg==} + '@web3-storage/content-claims@4.0.5': resolution: {integrity: sha512-+WpCkTN8aRfUCrCm0kOMZad+FRnFymVDFvS6/+PJMPGP17cci1/c5lqYdrjFV+5MkhL+BkUJVtRTx02G31FHmQ==} @@ -2532,9 +2548,23 @@ packages: '@web3-storage/data-segment@5.3.0': resolution: {integrity: sha512-zFJ4m+pEKqtKatJNsFrk/2lHeFSbkXZ6KKXjBe7/2ayA9wAar7T/unewnOcZrrZTnCWmaxKsXWqdMFy9bXK9dw==} + '@web3-storage/did-mailto@2.1.0': + resolution: {integrity: sha512-TRmfSXj1IhtX3ESurSNOylZSBKi0z/VJNoMLpof+AVRdovgZjjocpiePQTs2pfHKqHTHfJXc9AboWyK4IKTWMw==} + engines: {node: '>=16.15'} + + '@web3-storage/filecoin-client@3.3.4': + resolution: {integrity: sha512-T2xur1NPvuH09yajyjCWEl7MBH712nqHERj51w4nDp6f8libMCKY6lca0frCrm4OC5s8oy0ZtoRFhsRYxgTzSg==} + '@web3-storage/sigv4@1.0.2': resolution: {integrity: sha512-ZUXKK10NmuQgPkqByhb1H3OQxkIM0CIn2BMPhGQw7vQw8WIzrBkk9IJiAVfJ/UVBFrf6uzPbx2lEBLt4diCMnQ==} + '@web3-storage/upload-client@17.1.0': + resolution: {integrity: sha512-0tUMe4Ez9gmUZjgn1Nrl6HYdGEsYyeLa6JrpoXcCGTQDBW2FehALc+GZZeoIjYQexRpw+qt9JstuJNN9dUNETw==} + + '@web3-storage/w3up-client@16.5.1': + resolution: {integrity: sha512-tsBUSo6/bPElGAlMbHVeQDMgg4krRrKXOBl0TZttAAIYcMzTvm7fkDfD9+5rvJFsa/a846XqPoxtO21LwL768A==} + engines: {node: '>=18'} + '@webassemblyjs/ast@1.14.1': resolution: {integrity: sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==} @@ -10168,6 +10198,48 @@ snapshots: dependencies: web-streams-polyfill: 3.3.3 + '@web3-storage/access@20.1.0': + dependencies: + '@ipld/car': 5.3.3 + '@ipld/dag-ucan': 3.4.0 + '@scure/bip39': 1.4.0 + '@storacha/one-webcrypto': 1.0.1 + '@ucanto/client': 9.0.1 + '@ucanto/core': 10.0.1 + '@ucanto/interface': 10.0.1 + '@ucanto/principal': 9.0.1 + '@ucanto/transport': 9.1.1 + '@ucanto/validator': 9.0.2 + '@web3-storage/capabilities': 17.4.1 + '@web3-storage/did-mailto': 2.1.0 + bigint-mod-arith: 3.3.1 + conf: 11.0.2 + multiformats: 12.1.3 + p-defer: 4.0.1 + type-fest: 4.26.1 + uint8arrays: 4.0.10 + + '@web3-storage/blob-index@1.0.4': + dependencies: + '@ipld/dag-cbor': 9.2.2 + '@storacha/one-webcrypto': 1.0.1 + '@ucanto/core': 10.0.1 + '@ucanto/interface': 10.0.1 + '@web3-storage/capabilities': 17.4.1 + carstream: 2.2.0 + multiformats: 13.3.1 + uint8arrays: 5.1.0 + + '@web3-storage/capabilities@17.4.1': + dependencies: + '@ucanto/core': 10.0.1 + '@ucanto/interface': 10.0.1 + '@ucanto/principal': 9.0.1 + '@ucanto/transport': 9.1.1 + '@ucanto/validator': 9.0.2 + '@web3-storage/data-segment': 5.3.0 + uint8arrays: 5.1.0 + '@web3-storage/content-claims@4.0.5': dependencies: '@ucanto/client': 9.0.1 @@ -10198,10 +10270,59 @@ snapshots: multiformats: 13.3.1 sync-multihash-sha2: 1.0.0 + '@web3-storage/did-mailto@2.1.0': {} + + '@web3-storage/filecoin-client@3.3.4': + dependencies: + '@ipld/dag-ucan': 3.4.0 + '@ucanto/client': 9.0.1 + '@ucanto/core': 10.0.1 + '@ucanto/interface': 10.0.1 + '@ucanto/transport': 9.1.1 + '@web3-storage/capabilities': 17.4.1 + '@web3-storage/sigv4@1.0.2': dependencies: '@noble/hashes': 1.5.0 + '@web3-storage/upload-client@17.1.0': + dependencies: + '@ipld/car': 5.3.3 + '@ipld/dag-cbor': 9.2.2 + '@ipld/dag-ucan': 3.4.0 + '@ipld/unixfs': 2.2.0 + '@ucanto/client': 9.0.1 + '@ucanto/core': 10.0.1 + '@ucanto/interface': 10.0.1 + '@ucanto/transport': 9.1.1 + '@web3-storage/blob-index': 1.0.4 + '@web3-storage/capabilities': 17.4.1 + '@web3-storage/data-segment': 5.3.0 + '@web3-storage/filecoin-client': 3.3.4 + ipfs-utils: 9.0.14(encoding@0.1.13) + multiformats: 12.1.3 + p-retry: 5.1.2 + varint: 6.0.0 + transitivePeerDependencies: + - encoding + + '@web3-storage/w3up-client@16.5.1': + dependencies: + '@ipld/dag-ucan': 3.4.0 + '@ucanto/client': 9.0.1 + '@ucanto/core': 10.0.1 + '@ucanto/interface': 10.0.1 + '@ucanto/principal': 9.0.1 + '@ucanto/transport': 9.1.1 + '@web3-storage/access': 20.1.0 + '@web3-storage/blob-index': 1.0.4 + '@web3-storage/capabilities': 17.4.1 + '@web3-storage/did-mailto': 2.1.0 + '@web3-storage/filecoin-client': 3.3.4 + '@web3-storage/upload-client': 17.1.0 + transitivePeerDependencies: + - encoding + '@webassemblyjs/ast@1.14.1': dependencies: '@webassemblyjs/helper-numbers': 1.13.2