Skip to content

Commit

Permalink
fix: compatibility with legacy w3up client (#59)
Browse files Browse the repository at this point in the history
This PR allows `upload-service` to work with the legacy client
(`@web3-storage/w3up-client`).

The new service uses `blob/allocate` and `blob/accept` capabilities when
talking to storage nodes. For some reason I thought these were not
referenced by the client but they are....except via their old names
`web3.storage/blob/allocate` and `web3.storage/blob/accept` 🤦

This PR adds additional effects to the `blob/add` receipt, essentially
"fake" `web3.storage/blob/allocate` and `web3.storage/blob/accept`
delegations that contain the same information as their `blob/allocate`
and `blob/accept` counterparts. It allows old clients to use the old
capability names and the new clients to use the new ones.

We can remove these additional effects when we are satisfied enough
clients have moved to `@storacha/client`.
  • Loading branch information
alanshaw authored Nov 27, 2024
1 parent eaa5572 commit 7185046
Show file tree
Hide file tree
Showing 9 changed files with 364 additions and 11 deletions.
29 changes: 27 additions & 2 deletions packages/upload-api/src/blob/accept.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import * as Blob from '@storacha/capabilities/blob'
import { Message, Invocation } from '@ucanto/core'
import * as W3sBlob from '@storacha/capabilities/web3.storage/blob'
import { Message, Receipt, Invocation } from '@ucanto/core'
import * as Transport from '@ucanto/transport/car'
import * as API from '../types.js'
import * as HTTP from '@storacha/capabilities/http'
Expand Down Expand Up @@ -83,10 +84,34 @@ export const poll = async (context, receipt) => {
configure.ok.connection
)

// Create receipt for legacy `web3.storage/blob/accept`. The old client
// `@web3-storage/w3up-client` will poll for a receipt for this task, so
// we create one whose result is simply the result of the actual `blob/accept`
// task.
//
// TODO: remove when all users migrate to `@storacha/client`.
const w3sAccept = W3sBlob.accept.invoke({
issuer: context.id,
audience: context.id,
with: context.id.did(),
nb: {
blob: allocate.nb.blob,
space: /** @type {API.DIDKey} */ (DID.decode(allocate.nb.space).did()),
_put: { 'ucan/await': ['.out.ok', receipt.ran.link()] },
},
})
const w3sAcceptTask = await w3sAccept.delegate()
const w3sAcceptReceipt = await Receipt.issue({
issuer: context.id,
ran: w3sAcceptTask.cid,
result: acceptReceipt.out,
fx: acceptReceipt.fx,
})

// record the invocation and the receipt
const message = await Message.build({
invocations: [configure.ok.invocation],
receipts: [acceptReceipt],
receipts: [acceptReceipt, w3sAcceptReceipt],
})
const messageWrite = await context.agentStore.messages.write({
source: await Transport.outbound.encode(message),
Expand Down
136 changes: 134 additions & 2 deletions packages/upload-api/src/blob/add.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import * as Transport from '@ucanto/transport/car'
import { ed25519 } from '@ucanto/principal'
import * as Blob from '@storacha/capabilities/blob'
import * as SpaceBlob from '@storacha/capabilities/space/blob'
import * as W3sBlob from '@storacha/capabilities/web3.storage/blob'
import * as HTTP from '@storacha/capabilities/http'
import * as Digest from 'multiformats/hashes/digest'
import * as DID from '@ipld/dag-ucan/did'
Expand Down Expand Up @@ -50,6 +51,14 @@ export function blobAddProvider(context) {
return allocation
}

const allocationW3s = await allocateW3s({
context,
blob,
space,
cause: invocation.link(),
receipt: allocation.ok.receipt,
})

const delivery = await put({
blob,
allocation: allocation.ok,
Expand All @@ -66,20 +75,37 @@ export function blobAddProvider(context) {
return acceptance
}

// Create a result describing the this invocation workflow
const acceptanceW3s = await acceptW3s({
context,
provider: allocation.ok.provider,
blob,
space,
delivery: delivery,
acceptance: acceptance.ok,
})

// Create a result describing this invocation workflow
let result = Server.ok({
/** @type {API.SpaceBlobAddSuccess['site']} */
site: {
'ucan/await': ['.out.ok.site', acceptance.ok.task.link()],
},
})
.fork(allocation.ok.task)
.fork(allocationW3s.task)
.fork(delivery.task)
.fork(acceptance.ok.task)
.fork(acceptanceW3s.task)

// As a temporary solution we fork all add effects that add inline
// receipts so they can be delivered to the client.
const fx = [...allocation.ok.fx, ...delivery.fx, ...acceptance.ok.fx]
const fx = [
...allocation.ok.fx,
...allocationW3s.fx,
...delivery.fx,
...acceptance.ok.fx,
...acceptanceW3s.fx,
]
for (const task of fx) {
result = result.fork(task)
}
Expand Down Expand Up @@ -172,6 +198,49 @@ async function allocate({ context, blob, space, cause }) {
})
}

/**
* Create an allocation task and receipt using the legacy
* `web3.storage/blob/allocate` capability. This enables backwards compatibility
* with `@web3-storage/w3up-client`.
*
* TODO: remove when all users migrate to `@storacha/client`.
*
* @param {object} allocate
* @param {API.BlobServiceContext} allocate.context
* @param {API.BlobModel} allocate.blob
* @param {API.DIDKey} allocate.space
* @param {API.Link} allocate.cause
* @param {API.Receipt<API.BlobAllocateSuccess, API.BlobAcceptFailure>} allocate.receipt
*/
async function allocateW3s({ context, blob, space, cause, receipt }) {
const w3sAllocate = W3sBlob.allocate.invoke({
issuer: context.id,
audience: context.id,
with: context.id.did(),
nb: { blob, cause, space },
expiration: Infinity,
})
const w3sAllocateTask = await w3sAllocate.delegate()

const w3sAllocateReceipt = await Receipt.issue({
issuer: context.id,
ran: w3sAllocateTask.cid,
result: receipt.out,
})

const w3sAllocateConclude = createConcludeInvocation(
context.id,
context.id,
w3sAllocateReceipt
)

return {
task: w3sAllocateTask,
receipt: w3sAllocateReceipt,
fx: [await w3sAllocateConclude.delegate()],
}
}

/**
* Create put task and check if there is a receipt for it already.
* A `http/put` should be task is stored by the service, if it does not exist
Expand Down Expand Up @@ -313,3 +382,66 @@ async function accept({ context, provider, blob, space, delivery }) {
fx: receipt ? [await conclude(receipt, context.id)] : [],
})
}

/**
* Create an accept task and receipt using the legacy
* `web3.storage/blob/accept` capability. This enables backwards compatibility
* with `@web3-storage/w3up-client`.
*
* TODO: remove when all users migrate to `@storacha/client`.
*
* @param {object} input
* @param {API.BlobServiceContext} input.context
* @param {API.Principal} input.provider
* @param {API.BlobModel} input.blob
* @param {API.DIDKey} input.space
* @param {object} input.delivery
* @param {API.Invocation<API.HTTPPut>} input.delivery.task
* @param {API.Receipt|null} input.delivery.receipt
* @param {object} input.acceptance
* @param {API.Receipt|null} input.acceptance.receipt
*/
async function acceptW3s({ context, blob, space, delivery, acceptance }) {
// 1. Create web3.storage/blob/accept invocation and task
const w3sAccept = W3sBlob.accept.invoke({
issuer: context.id,
audience: context.id,
with: context.id.did(),
nb: {
blob,
space,
_put: { 'ucan/await': ['.out.ok', delivery.task.link()] },
},
})
const w3sAcceptTask = await w3sAccept.delegate()

let w3sAcceptReceipt = null
// If put has failed, we propagate the error to the `blob/accept` receipt.
if (delivery.receipt?.out.error) {
w3sAcceptReceipt = await Receipt.issue({
issuer: context.id,
ran: w3sAcceptTask,
result: {
error: new AwaitError({
cause: delivery.receipt.out.error,
at: '.out.ok',
reference: delivery.task.link(),
}),
},
})
}
// If `blob/accept` receipt is present, we issue a receipt for
// `web3.storage/blob/accept`.
else if (acceptance.receipt) {
w3sAcceptReceipt = await Receipt.issue({
issuer: context.id,
ran: w3sAcceptTask,
result: acceptance.receipt.out,
})
}

return {
task: w3sAcceptTask,
fx: w3sAcceptReceipt ? [await conclude(w3sAcceptReceipt, context.id)] : [],
}
}
2 changes: 2 additions & 0 deletions packages/upload-api/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,8 @@ export interface RevocationServiceContext {
}

export interface ConcludeServiceContext {
/** Upload service signer. */
id: Signer
/**
* Store for invocations & receipts.
*/
Expand Down
2 changes: 1 addition & 1 deletion packages/upload-api/test/handlers/blob.js
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ export const test = {
blobAdd.out.ok.site['ucan/await'][1],
next.accept.task.cid
)
assert.equal(blobAdd.fx.fork.length, 4)
assert.equal(blobAdd.fx.fork.length, 7)

// validate receipt next
assert.ok(next.allocate.task)
Expand Down
12 changes: 7 additions & 5 deletions packages/upload-client/src/blob/add.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { ed25519 } from '@ucanto/principal'
import { conclude } from '@storacha/capabilities/ucan'
import * as UCAN from '@storacha/capabilities/ucan'
import { Delegation, Receipt } from '@ucanto/core'
import * as BlobCapabilities from '@storacha/capabilities/blob'
Expand Down Expand Up @@ -121,7 +120,7 @@ export function createConcludeInvocation(id, serviceDid, receipt) {
receiptBlocks.push(block)
receiptCids.push(block.cid)
}
const concludeAllocatefx = conclude.invoke({
const concludeAllocatefx = UCAN.conclude.invoke({
issuer: id,
audience: serviceDid,
with: id.toDIDKey(),
Expand Down Expand Up @@ -292,9 +291,12 @@ export async function add(
)
const ucanConclude = await httpPutConcludeInvocation.execute(conn)
if (!ucanConclude.out.ok) {
throw new Error(`failed ${SpaceBlobCapabilities.add.can} invocation`, {
cause: result.out.error,
})
throw new Error(
`failed ${UCAN.conclude.can} for ${HTTPCapabilities.put.can} invocation`,
{
cause: result.out.error,
}
)
}
}

Expand Down
2 changes: 1 addition & 1 deletion packages/upload-client/test/blob.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ describe('Blob.add', () => {
{ connection }
),
{
message: 'failed space/blob/add invocation',
message: 'failed ucan/conclude for http/put invocation',
}
)
})
Expand Down
2 changes: 2 additions & 0 deletions packages/w3up-client/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,10 @@
"@types/mocha": "^10.0.1",
"@types/node": "^20.8.4",
"@ucanto/server": "^10.0.0",
"@web3-storage/access": "^20.1.0",
"@web3-storage/content-claims": "^4.0.4",
"@web3-storage/data-segment": "^5.0.0",
"@web3-storage/w3up-client": "^16.5.1",
"assert": "^2.0.0",
"c8": "^7.13.0",
"hundreds": "^0.0.9",
Expand Down
69 changes: 69 additions & 0 deletions packages/w3up-client/test/legacy-compat.node.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import http from 'node:http'
import { Client } from '@web3-storage/w3up-client'
import { AgentData } from '@web3-storage/access'
import * as Link from 'multiformats/link'
import { Message } from '@ucanto/core'
import * as CAR from '@ucanto/transport/car'
import * as Test from './test.js'
import { randomBytes } from './helpers/random.js'

/** @param {import('@storacha/upload-api').AgentStore} agentStore */
const createReceiptsServer = (agentStore) =>
http.createServer(async (req, res) => {
const task = Link.parse(req.url?.split('/').pop() ?? '')
const receiptGet = await agentStore.receipts.get(task)
if (receiptGet.error) {
res.writeHead(404)
return res.end()
}
const message = await Message.build({ receipts: [receiptGet.ok] })
const request = CAR.request.encode(message)
res.writeHead(200)
res.end(request.body)
})

/** @type {Test.Suite} */
export const testLegacyCompatibility = {
uploadFile: Test.withContext({
'should upload a file to the service via legacy client': async (
assert,
{ connection, provisionsStorage, agentStore }
) => {
const receiptsServer = createReceiptsServer(agentStore)
const receiptsEndpoint = await new Promise((resolve) => {
receiptsServer.listen(() => {
// @ts-expect-error
resolve(new URL(`http://127.0.0.1:${receiptsServer.address().port}`))
})
})

try {
const bytes = await randomBytes(128)
const file = new Blob([bytes])
const alice = new Client(await AgentData.create(), {
// @ts-expect-error service no longer implements `store/*`
serviceConf: { access: connection, upload: connection },
receiptsEndpoint,
})

const space = await alice.createSpace('upload-test')
const auth = await space.createAuthorization(alice)
await alice.addSpace(auth)
await alice.setCurrentSpace(space.did())

await provisionsStorage.put({
// @ts-expect-error
provider: connection.id.did(),
account: alice.agent.did(),
consumer: space.did(),
})

await assert.doesNotReject(alice.uploadFile(file))
} finally {
receiptsServer.close()
}
},
}),
}

Test.test({ LegacyCompatibility: testLegacyCompatibility })
Loading

0 comments on commit 7185046

Please sign in to comment.