feat: move blob index logic from upload-api to blob-index lib (storac…

…ha#1434)

The blob index logic should be an independent module as it will be used
in both client and service code.

---------

Co-authored-by: Alan Shaw <[email protected]>

.github/release-please-config.json

@@ -8,6 +8,7 @@
   ],
   "packages": {
     "packages/access-client": {},
+    "packages/blob-index": {},
     "packages/filecoin-api": {},
     "packages/filecoin-client": {},
     "packages/capabilities": {},

.github/release-please-manifest.json

@@ -1,5 +1,6 @@
 {
   "packages/access-client": "18.3.2",
+  "packages/blob-index": "0.0.0",
   "packages/filecoin-api": "6.0.1",
   "packages/filecoin-client": "3.3.3",
   "packages/capabilities": "17.0.0",

.github/workflows/blob-index.yml

@@ -0,0 +1,39 @@
+name: blob-index
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/blob-index/**'
+      - 'packages/eslint-config-w3up/**'
+      - '.github/workflows/blob-index.yml'
+      - 'pnpm-lock.yaml'
+  pull_request:
+    paths:
+      - 'packages/blob-index/**'
+      - 'packages/eslint-config-w3up/**'
+      - '.github/workflows/blob-index.yml'
+      - 'pnpm-lock.yaml'
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./packages/blob-index
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install
+        uses: pnpm/[email protected]
+        with:
+          version: 8
+      - name: Setup
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+          cache: 'pnpm'
+      - run: pnpm --filter '@web3-storage/blob-index...' install
+      - run: pnpm --filter '@web3-storage/blob-index' lint
+      - run: pnpm --filter '@web3-storage/blob-index' test

packages/blob-index/README.md

@@ -0,0 +1,62 @@
+# `@web3-storage/blob-index`
+
+An index for slices that may be sharded across multiple blobs.
+
+## Install
+
+```
+npm i @web3-storage/blob-index
+```
+
+## Usage
+
+Create:
+
+```js
+import { ShardedDAGIndex } from '@web3-storage/blob-index'
+
+// Create a brand new index
+const index = ShardedDAGIndex.create(rootCID)
+
+// Add index data for slices within a shard
+index.setSlice(shardMultihash, sliceMultihash, [offset, length])
+// ...
+
+// Create CAR archive
+const result = index.archive()
+
+console.log(result.ok) // a Uint8Array
+```
+
+Read:
+
+```js
+import { ShardedDAGIndex } from '@web3-storage/blob-index'
+import { base58btc } from 'multiformats/bases/base58'
+
+const index = ShardedDAGIndex.extract(car)
+
+console.log(index.content)
+
+for (const [shard, slices] of index.shards.entries()) {
+  console.log(`Shard ${base58btc.encode(shard.bytes)}`)
+  console.log('  Slices:')
+  for (const [slice, [offset, length]] of slices.entries()) {
+    console.log(`    ${base58btc.encode(slice.bytes)} @ ${offset} -> ${offset + length}`)
+  }
+}
+
+// Output:
+// Shard zQmQKw6B745GGL3eeTcEE5kAoLAJgkBQydJPC5fWv5HA68A
+//  Slices:
+//    zQmeHPRNRDxHU5YMPewcBCbPYxzA3jBcadAZQwpQXm3jFFt @ 96 -> 128
+// ...
+```
+
+## Contributing
+
+Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3up/issues)!
+
+## License
+
+Dual-licensed under [MIT + Apache 2.0](https://github.com/web3-storage/w3up/blob/main/license.md)

packages/blob-index/package.json

@@ -0,0 +1,81 @@
+{
+  "name": "@web3-storage/blob-index",
+  "description": "An index for slices that may be sharded across multiple blobs.",
+  "version": "0.0.0",
+  "homepage": "https://web3.storage",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/w3s-project/w3up.git",
+    "directory": "packages/blob-index"
+  },
+  "license": "Apache-2.0 OR MIT",
+  "type": "module",
+  "types": "dist/src/index.d.ts",
+  "main": "src/index.js",
+  "files": [
+    "src",
+    "test",
+    "dist"
+  ],
+  "exports": {
+    ".": "./dist/src/index.js",
+    "./sharded-dag-index": "./dist/src/sharded-dag-index.js",
+    "./types": "./dist/src/api.js",
+    "./util": "./dist/src/util.js"
+  },
+  "typesVersions": {
+    "*": {
+      "types": [
+        "dist/src/types"
+      ]
+    }
+  },
+  "scripts": {
+    "attw": "attw --pack .",
+    "build": "tsc --build",
+    "check": "tsc --build",
+    "lint": "tsc --build && eslint '**/*.{js,ts}' && prettier --check '**/*.{js,ts,yml,json}' --ignore-path ../../.gitignore",
+    "test": "entail test/*.spec.js",
+    "coverage": "c8 --reporter text --reporter html npm run test"
+  },
+  "dependencies": {
+    "@ipld/dag-cbor": "^9.0.6",
+    "@ucanto/core": "^10.0.1",
+    "@ucanto/interface": "^10.0.1",
+    "@web3-storage/capabilities": "workspace:^",
+    "multiformats": "^13.0.1",
+    "uint8arrays": "^5.0.3"
+  },
+  "devDependencies": {
+    "@ipld/car": "^5.1.1",
+    "@types/assert": "^1.5.6",
+    "@types/mocha": "^10.0.1",
+    "@ucanto/transport": "^9.1.1",
+    "@web-std/blob": "^3.0.5",
+    "@web3-storage/eslint-config-w3up": "workspace:^",
+    "c8": "^7.14.0",
+    "carstream": "^2.1.0",
+    "entail": "^2.1.2",
+    "one-webcrypto": "git://github.com/web3-storage/one-webcrypto",
+    "typescript": "5.2.2"
+  },
+  "eslintConfig": {
+    "extends": [
+      "@web3-storage/eslint-config-w3up"
+    ],
+    "parserOptions": {
+      "project": "./tsconfig.json"
+    },
+    "env": {
+      "mocha": true
+    },
+    "ignorePatterns": [
+      "dist",
+      "coverage",
+      "src/types.js"
+    ]
+  },
+  "engines": {
+    "node": ">=16.15"
+  }
+}

packages/blob-index/src/api.ts

@@ -0,0 +1,33 @@
+import { Result, Failure } from '@ucanto/interface'
+import { MultihashDigest, Link, UnknownLink } from 'multiformats'
+
+export type { IPLDBlock } from '@ucanto/interface'
+export type { UnknownFormat } from '@web3-storage/capabilities/types'
+export type { Result, MultihashDigest, Link, UnknownLink }
+
+export type ShardDigest = MultihashDigest
+export type SliceDigest = MultihashDigest
+export type Position = [offset: number, length: number]
+
+/**
+ * A sharded DAG index.
+ *
+ * @see https://github.com/w3s-project/specs/blob/main/w3-index.md
+ */
+export interface ShardedDAGIndex {
+  /** DAG root CID that the index pertains to. */
+  content: UnknownLink
+  /** Index information for shards the DAG is split across. */
+  shards: Map<ShardDigest, Map<SliceDigest, Position>>
+}
+
+export interface ShardedDAGIndexView extends ShardedDAGIndex {
+  /** Set the offset/length information for the slice a shard. */
+  setSlice(shard: ShardDigest, slice: SliceDigest, pos: Position): void
+  /** Encode the index to a CAR file. */
+  archive(): Promise<Result<Uint8Array>>
+}
+
+export interface DecodeFailure extends Failure {
+  name: 'DecodeFailure'
+}

packages/upload-api/src/index/lib/digest-map.js → packages/blob-index/src/digest-map.js

@@ -1,9 +1,10 @@
+import * as API from './api.js'
 import { base58btc } from 'multiformats/bases/base58'
 
 /** @type {WeakMap<Uint8Array, string>} */
 const cache = new WeakMap()
 
-/** @param {import('multiformats').MultihashDigest} digest */
+/** @param {API.MultihashDigest} digest */
 const toBase58String = (digest) => {
   let str = cache.get(digest.bytes)
   if (!str) {
@@ -14,7 +15,7 @@ const toBase58String = (digest) => {
 }
 
 /**
- * @template {import('multiformats').MultihashDigest} Key
+ * @template {API.MultihashDigest} Key
  * @template Value
  * @implements {Map<Key, Value>}
  */

packages/blob-index/src/index.js

@@ -0,0 +1,2 @@
+export * as ShardedDAGIndex from './sharded-dag-index.js'
+export * from './digest-map.js'