From 5779161e349a752ee0c08fc32ade895cb56a1594 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 24 Apr 2024 17:07:26 +0100 Subject: [PATCH] feat(billing): support blob capabilities (#356) * Use the blob allocate receipt to track space size additions. * Use blob remove receipt to track space size reductions. --- billing/lib/ucan-stream.js | 42 ++++++++++++++++-- billing/package.json | 2 +- billing/test/helpers/did.js | 7 ++- billing/test/lib/ucan-stream.js | 76 +++++++++++++++++++++++++++------ package-lock.json | 43 ++++++++++++++++++- 5 files changed, 151 insertions(+), 19 deletions(-) diff --git a/billing/lib/ucan-stream.js b/billing/lib/ucan-stream.js index 0ce584e9..a2e1643c 100644 --- a/billing/lib/ucan-stream.js +++ b/billing/lib/ucan-stream.js @@ -1,3 +1,5 @@ +import * as ServiceBlobCaps from '@web3-storage/capabilities/web3.storage/blob' +import * as BlobCaps from '@web3-storage/capabilities/blob' import * as StoreCaps from '@web3-storage/capabilities/store' /** @@ -13,23 +15,35 @@ export const findSpaceUsageDeltas = messages => { for (const message of messages) { if (!isReceipt(message)) continue + /** @type {import('@ucanto/interface').DID|undefined} */ + let resource /** @type {number|undefined} */ let size - if (isReceiptForCapability(message, StoreCaps.add) && isStoreAddSuccess(message.out)) { + if (isReceiptForCapability(message, ServiceBlobCaps.allocate) && isServiceBlobAllocateSuccess(message.out)) { + resource = message.value.att[0].nb?.space + size = message.out.ok.size + } else if (isReceiptForCapability(message, BlobCaps.remove) && isBlobRemoveSuccess(message.out)) { + resource = /** @type {import('@ucanto/interface').DID} */ (message.value.att[0].with) + size = -message.out.ok.size + // TODO: remove me LEGACY store/add + } else if (isReceiptForCapability(message, StoreCaps.add) && isStoreAddSuccess(message.out)) { + resource = /** @type {import('@ucanto/interface').DID} */ (message.value.att[0].with) size = message.out.ok.allocated + // TODO: remove me LEGACY store/remove } else if (isReceiptForCapability(message, StoreCaps.remove) && isStoreRemoveSuccess(message.out)) { + resource = /** @type {import('@ucanto/interface').DID} */ (message.value.att[0].with) size = -message.out.ok.size } // Is message is a repeat store/add for the same shard or not a valid // store/add or store/remove receipt? - if (size == 0 || size == null) { + if (resource == null || size == 0 || size == null) { continue } /** @type {import('./api.js').UsageDelta} */ const delta = { - resource: /** @type {import('@ucanto/interface').DID} */ (message.value.att[0].with), + resource, cause: message.invocationCid, delta: size, // TODO: use receipt timestamp per https://github.com/web3-storage/w3up/issues/970 @@ -83,6 +97,28 @@ export const storeSpaceUsageDelta = async (delta, ctx) => { */ const isReceipt = m => m.type === 'receipt' +/** + * @param {import('@ucanto/interface').Result} r + * @returns {r is { ok: import('@web3-storage/capabilities/types').BlobAllocateSuccess }} + */ +const isServiceBlobAllocateSuccess = r => + !r.error && + r.ok != null && + typeof r.ok === 'object' && + 'size' in r.ok && + (typeof r.ok.size === 'number') + +/** + * @param {import('@ucanto/interface').Result} r + * @returns {r is { ok: import('@web3-storage/capabilities/types').BlobRemoveSuccess }} + */ +const isBlobRemoveSuccess = r => + !r.error && + r.ok != null && + typeof r.ok === 'object' && + 'size' in r.ok && + (typeof r.ok.size === 'number') + /** * @param {import('@ucanto/interface').Result} r * @returns {r is { ok: import('@web3-storage/capabilities/types').StoreAddSuccess }} diff --git a/billing/package.json b/billing/package.json index 18dde34c..59f864f9 100644 --- a/billing/package.json +++ b/billing/package.json @@ -13,7 +13,7 @@ "@sentry/serverless": "^7.74.1", "@ucanto/interface": "^10.0.1", "@ucanto/server": "^10.0.0", - "@web3-storage/capabilities": "^13.3.1", + "@web3-storage/capabilities": "^14.0.0", "big.js": "^6.2.1", "multiformats": "^13.1.0", "p-retry": "^6.2.0", diff --git a/billing/test/helpers/did.js b/billing/test/helpers/did.js index bc3d3373..c58fb9e7 100644 --- a/billing/test/helpers/did.js +++ b/billing/test/helpers/did.js @@ -11,8 +11,11 @@ const randomDomain = () => export const randomDIDMailto = () => `did:mailto:${randomDomain()}:${randomAlphas(randomInteger(1, 16))}` - /** @returns {Promise} */ -export const randomDID = async () => { +/** @returns {Promise} */ +export const randomDID = () => randomDIDKey() + +/** @returns {Promise>} */ +export const randomDIDKey = async () => { const signer = await Signer.generate() return signer.did() } diff --git a/billing/test/lib/ucan-stream.js b/billing/test/lib/ucan-stream.js index 2bd27dae..6c4f2129 100644 --- a/billing/test/lib/ucan-stream.js +++ b/billing/test/lib/ucan-stream.js @@ -1,8 +1,11 @@ import { Schema } from '@ucanto/core' +import * as ServiceBlobCaps from '@web3-storage/capabilities/web3.storage/blob' +import * as BlobCaps from '@web3-storage/capabilities/blob' +import * as StoreCaps from '@web3-storage/capabilities/store' import { findSpaceUsageDeltas, storeSpaceUsageDelta } from '../../lib/ucan-stream.js' import { randomConsumer } from '../helpers/consumer.js' import { randomLink } from '../helpers/dag.js' -import { randomDID } from '../helpers/did.js' +import { randomDID, randomDIDKey } from '../helpers/did.js' /** @type {import('./api').TestSuite} */ export const test = { @@ -14,7 +17,7 @@ export const test = { value: { att: [{ with: await randomDID(), - can: 'store/list' + can: StoreCaps.list.can }], aud: await randomDID(), cid: randomLink() @@ -24,15 +27,61 @@ export const test = { const shard = randomLink() - /** @type {import('../../lib/api.js').UcanReceiptMessage[]} */ + /** + * @type {import('../../lib/api.js').UcanReceiptMessage<[ + * | import('@web3-storage/capabilities/types').BlobAllocate + * | import('@web3-storage/capabilities/types').BlobRemove + * | import('@web3-storage/capabilities/types').StoreAdd + * | import('@web3-storage/capabilities/types').StoreRemove + * ]>[]} + */ const receipts = [{ type: 'receipt', carCid: randomLink(), invocationCid: randomLink(), value: { att: [{ - with: await randomDID(), - can: 'store/add', + with: await randomDIDKey(), + can: ServiceBlobCaps.allocate.can, + nb: { + blob: { + digest: randomLink().multihash.bytes, + size: 138 + }, + cause: randomLink(), + space: await randomDIDKey() + } + }], + aud: await randomDID(), + cid: randomLink() + }, + out: { ok: { size: 138 } }, + ts: new Date() + }, { + type: 'receipt', + carCid: randomLink(), + invocationCid: randomLink(), + value: { + att: [{ + with: await randomDIDKey(), + can: BlobCaps.remove.can, + nb: { + digest: randomLink().multihash.bytes + } + }], + aud: await randomDID(), + cid: randomLink() + }, + out: { ok: { size: 138 } }, + ts: new Date() + }, { + type: 'receipt', + carCid: randomLink(), + invocationCid: randomLink(), + value: { + att: [{ + with: await randomDIDKey(), + can: StoreCaps.add.can, nb: { link: shard, size: 138 @@ -49,8 +98,8 @@ export const test = { invocationCid: randomLink(), value: { att: [{ - with: await randomDID(), - can: 'store/remove', + with: await randomDIDKey(), + can: StoreCaps.remove.can, nb: { link: shard } }], aud: await randomDID(), @@ -66,8 +115,11 @@ export const test = { // ensure we have a delta for every receipt for (const r of receipts) { assert.ok(deltas.some(d => ( - d.resource === r.value.att[0].with && - d.cause.toString() === r.invocationCid.toString() + d.cause.toString() === r.invocationCid.toString() && + // resource for blob allocate is found in the caveats + (r.value.att[0].can === ServiceBlobCaps.allocate.can + ? d.resource === r.value.att[0].nb.space + : d.resource === r.value.att[0].with) ))) } }, @@ -86,7 +138,7 @@ export const test = { value: { att: [{ with: Schema.did({ method: 'key' }).from(consumer.consumer), - can: 'store/add', + can: StoreCaps.add.can, nb: { link: randomLink(), size: 138 @@ -104,7 +156,7 @@ export const test = { value: { att: [{ with: Schema.did({ method: 'key' }).from(consumer.consumer), - can: 'store/add', + can: StoreCaps.add.can, nb: { link: randomLink(), size: 1138 @@ -158,7 +210,7 @@ export const test = { value: { att: [{ with: Schema.did({ method: 'key' }).from(consumer.consumer), - can: 'store/add', + can: StoreCaps.add.can, nb: { link: randomLink(), size: 138 diff --git a/package-lock.json b/package-lock.json index d9c9173b..b27e57b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -64,7 +64,7 @@ "@sentry/serverless": "^7.74.1", "@ucanto/interface": "^10.0.1", "@ucanto/server": "^10.0.0", - "@web3-storage/capabilities": "^13.3.1", + "@web3-storage/capabilities": "^14.0.0", "big.js": "^6.2.1", "multiformats": "^13.1.0", "p-retry": "^6.2.0", @@ -96,6 +96,47 @@ "@ucanto/validator": "^9.0.1" } }, + "billing/node_modules/@web3-storage/capabilities": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-14.0.0.tgz", + "integrity": "sha512-i57wEzIjBsz5iCdBJJZCCAN/j0Vknns2NDFEpV732Vo/VxW2PbYqVb0eqEKegDwxdAmkZwyTT6iZQfsebgK7hw==", + "dependencies": { + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" + } + }, + "billing/node_modules/@web3-storage/capabilities/node_modules/uint8arrays": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.0.3.tgz", + "integrity": "sha512-6LBuKji28kHjgPJMkQ6GDaBb1lRwIhyOYq6pDGwYMoDPfImE9SkuYENVmR0yu9yGgs2clHUSY9fKDukR+AXfqQ==", + "dependencies": { + "multiformats": "^13.0.0" + } + }, + "billing/node_modules/@web3-storage/data-segment": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.2.0.tgz", + "integrity": "sha512-SM6eNumXzrXiQE2/J59+eEgCRZNYPxKhRoHX2QvV3/scD4qgcf4g+paWBc3UriLEY1rCboygGoPsnqYJNyZyfA==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.5", + "multiformats": "^11.0.2", + "sync-multihash-sha2": "^1.0.0" + } + }, + "billing/node_modules/@web3-storage/data-segment/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "carpark": { "name": "@web3-storage/w3infra-carpark", "version": "0.0.0",