feat: update kzg to the latest decoupled and bigendian version

packages/beacon-node/package.json

@@ -133,7 +133,7 @@
     "@multiformats/multiaddr": "^11.0.0",
     "@types/datastore-level": "^3.0.0",
     "buffer-xor": "^2.0.2",
-    "c-kzg": "^1.0.9",
+    "c-kzg": "^2.1.0",
     "cross-fetch": "^3.1.4",
     "datastore-core": "^8.0.1",
     "datastore-level": "^9.0.1",

packages/beacon-node/src/chain/options.ts

@@ -21,6 +21,8 @@ export type IChainOptions = BlockProcessOpts &
     sanityCheckExecutionEngineBlobs?: boolean;
     /** Max number of produced blobs by local validators to cache */
     maxCachedBlobSidecars?: number;
+    /** Option to load a custom kzg trusted setup in txt format */
+    trustedSetup?: string;
   };
 
 export type BlockProcessOpts = {

packages/beacon-node/src/node/nodejs.ts

@@ -20,7 +20,7 @@ import {MonitoringService} from "../monitoring/index.js";
 import {getApi, BeaconRestApiServer} from "../api/index.js";
 import {initializeExecutionEngine, initializeExecutionBuilder} from "../execution/index.js";
 import {initializeEth1ForBlockProduction} from "../eth1/index.js";
-import {initCKZG, loadEthereumTrustedSetup} from "../util/kzg.js";
+import {initCKZG, loadEthereumTrustedSetup, TrustedFileMode} from "../util/kzg.js";
 import {IBeaconNodeOptions} from "./options.js";
 import {runNodeNotifier} from "./notifier.js";
 
@@ -161,7 +161,7 @@ export class BeaconNode {
       // TODO DENEB: "c-kzg" is not installed by default, so if the library is not installed this will throw
       // See "Not able to build lodestar from source" https://github.com/ChainSafe/lodestar/issues/4886
       await initCKZG();
-      loadEthereumTrustedSetup();
+      loadEthereumTrustedSetup(TrustedFileMode.Txt, opts.chain.trustedSetup);
     }
 
     // Prune hot db repos

packages/beacon-node/src/util/kzg.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import fs from "node:fs";
 import {fileURLToPath} from "node:url";
 import {fromHex, toHex} from "@lodestar/utils";
+import {ssz} from "@lodestar/types";
 
 /* eslint-disable @typescript-eslint/naming-convention */
 
@@ -16,6 +17,11 @@ export let ckzg: {
   freeTrustedSetup(): void;
   loadTrustedSetup(filePath: string): void;
   blobToKzgCommitment(blob: Uint8Array): Uint8Array;
+  computeBlobKzgProof(blob: Uint8Array, commitment: Uint8Array): Uint8Array;
+  verifyBlobKzgProof(blob: Uint8Array, commitment: Uint8Array, proof: Uint8Array): boolean;
+  verifyBlobKzgProofBatch(blobs: Uint8Array[], expectedKzgCommitments: Uint8Array[], kzgProofs: Uint8Array[]): boolean;
+
+  // TODO DENEB: Add these dummy methods to be removed on full transition to blobsidecars
   computeAggregateKzgProof(blobs: Uint8Array[]): Uint8Array;
   verifyAggregateKzgProof(
     blobs: Uint8Array[],
@@ -26,6 +32,11 @@ export let ckzg: {
   freeTrustedSetup: ckzgNotLoaded,
   loadTrustedSetup: ckzgNotLoaded,
   blobToKzgCommitment: ckzgNotLoaded,
+  computeBlobKzgProof: ckzgNotLoaded,
+  verifyBlobKzgProof: ckzgNotLoaded,
+  verifyBlobKzgProofBatch: ckzgNotLoaded,
+
+  // TODO DENEB: Add these dummy methods to be removed on full transition to blobsidecars
   computeAggregateKzgProof: ckzgNotLoaded,
   verifyAggregateKzgProof: ckzgNotLoaded,
 };
@@ -47,30 +58,51 @@ const TOTAL_SIZE = 2 * POINT_COUNT_BYTES + G1POINT_BYTES * G1POINT_COUNT + G2POI
 export async function initCKZG(): Promise<void> {
   /* eslint-disable import/no-extraneous-dependencies, @typescript-eslint/ban-ts-comment */
   // @ts-ignore
-  ckzg = (await import("c-kzg")) as typeof ckzg;
+  ckzg = (await import("c-kzg")).default as typeof ckzg;
   /* eslint-enable import/no-extraneous-dependencies, @typescript-eslint/ban-ts-comment */
+
+  // TODO DENEB: Add this dummy methods to be removed on full transition to blobsidecars
+  ckzg.computeAggregateKzgProof = (_blobs: Uint8Array[]) => {
+    return ssz.deneb.KZGProof.defaultValue();
+  };
+  ckzg.verifyAggregateKzgProof = (
+    _blobs: Uint8Array[],
+    _expectedKzgCommitments: Uint8Array[],
+    _kzgAggregatedProof: Uint8Array
+  ) => {
+    return true;
+  };
+}
+
+export enum TrustedFileMode {
+  Bin = "bin",
+  Txt = "txt",
 }
 
 /**
  * Load our KZG trusted setup into C-KZG for later use.
  * We persist the trusted setup as serialized bytes to save space over TXT or JSON formats.
  * However the current c-kzg API **requires** to read from a file with a specific .txt format
  */
-export function loadEthereumTrustedSetup(): void {
+export function loadEthereumTrustedSetup(mode: TrustedFileMode = TrustedFileMode.Txt, filePath?: string): void {
   try {
-    const bytes = fs.readFileSync(TRUSTED_SETUP_BIN_FILEPATH);
-    const json = trustedSetupBinToJson(bytes);
-    const txt = trustedSetupJsonToTxt(json);
-    fs.writeFileSync(TRUSTED_SETUP_TXT_FILEPATH, txt);
+    let setupFilePath;
+    if (mode === TrustedFileMode.Bin) {
+      const binPath = filePath ?? TRUSTED_SETUP_BIN_FILEPATH;
+      const bytes = fs.readFileSync(binPath);
+      const json = trustedSetupBinToJson(bytes);
+      const txt = trustedSetupJsonToTxt(json);
+      fs.writeFileSync(TRUSTED_SETUP_TXT_FILEPATH, txt);
+      setupFilePath = TRUSTED_SETUP_TXT_FILEPATH;
+    } else {
+      setupFilePath = filePath ?? TRUSTED_SETUP_TXT_FILEPATH;
+    }
 
     try {
       // in unit tests, calling loadTrustedSetup() twice has error so we have to free and retry
-      ckzg.loadTrustedSetup(TRUSTED_SETUP_TXT_FILEPATH);
+      ckzg.loadTrustedSetup(setupFilePath);
     } catch (e) {
-      if ((e as Error).message === "Call freeTrustedSetup before loading a new trusted setup.") {
-        ckzg.freeTrustedSetup();
-        ckzg.loadTrustedSetup(TRUSTED_SETUP_TXT_FILEPATH);
-      } else {
+      if ((e as Error).message !== "Error trusted setup is already loaded") {
         throw e;
       }
     }

packages/beacon-node/test/unit/util/kzg.test.ts

@@ -3,9 +3,18 @@ import {bellatrix, deneb, ssz} from "@lodestar/types";
 import {BYTES_PER_FIELD_ELEMENT, BLOB_TX_TYPE} from "@lodestar/params";
 import {kzgCommitmentToVersionedHash} from "@lodestar/state-transition";
 import {loadEthereumTrustedSetup, initCKZG, ckzg, FIELD_ELEMENTS_PER_BLOB_MAINNET} from "../../../src/util/kzg.js";
-import {validateBlobsSidecar, validateGossipBlobsSidecar} from "../../../src/chain/validation/blobsSidecar.js";
 
-describe("C-KZG", () => {
+import {getMockBeaconChain} from "../../utils/mocks/chain.js";
+
+describe("C-KZG", async () => {
+  const afterEachCallbacks: (() => Promise<unknown> | void)[] = [];
+  afterEach(async () => {
+    while (afterEachCallbacks.length > 0) {
+      const callback = afterEachCallbacks.pop();
+      if (callback) await callback();
+    }
+  });
+
   before(async function () {
     this.timeout(10000); // Loading trusted setup is slow
     await initCKZG();
@@ -18,34 +27,48 @@ describe("C-KZG", () => {
     // ====================
     const blobs = new Array(2).fill(0).map(generateRandomBlob);
     const commitments = blobs.map((blob) => ckzg.blobToKzgCommitment(blob));
-    const proof = ckzg.computeAggregateKzgProof(blobs);
-    expect(ckzg.verifyAggregateKzgProof(blobs, commitments, proof)).to.equal(true);
+    const proofs = blobs.map((blob, index) => ckzg.computeBlobKzgProof(blob, commitments[index]));
+    expect(ckzg.verifyBlobKzgProofBatch(blobs, commitments, proofs)).to.equal(true);
   });
 
-  it("BlobsSidecar", () => {
+  it("BlobSidecars", async () => {
+    const chain = getMockBeaconChain();
+    afterEachCallbacks.push(() => chain.close());
+
     const slot = 0;
     const blobs = [generateRandomBlob(), generateRandomBlob()];
     const kzgCommitments = blobs.map((blob) => ckzg.blobToKzgCommitment(blob));
 
     const signedBeaconBlock = ssz.deneb.SignedBeaconBlock.defaultValue();
+
     for (const kzgCommitment of kzgCommitments) {
       signedBeaconBlock.message.body.executionPayload.transactions.push(transactionForKzgCommitment(kzgCommitment));
       signedBeaconBlock.message.body.blobKzgCommitments.push(kzgCommitment);
     }
-    const beaconBlockRoot = ssz.deneb.BeaconBlock.hashTreeRoot(signedBeaconBlock.message);
+    const blockRoot = ssz.deneb.BeaconBlock.hashTreeRoot(signedBeaconBlock.message);
+
+    const blobSidecars: deneb.BlobSidecars = blobs.map((blob, index) => {
+      return {
+        blockRoot,
+        index,
+        slot,
+        blob,
+        kzgProof: ckzg.computeBlobKzgProof(blob, kzgCommitments[index]),
+        kzgCommitment: kzgCommitments[index],
+        blockParentRoot: Buffer.alloc(32),
+        proposerIndex: 0,
+      };
+    });
 
-    const blobsSidecar: deneb.BlobsSidecar = {
-      beaconBlockRoot,
-      beaconBlockSlot: 0,
-      blobs,
-      kzgAggregatedProof: ckzg.computeAggregateKzgProof(blobs),
-    };
+    const signedBlobSidecars: deneb.SignedBlobSidecar[] = blobSidecars.map((blobSidecar) => {
+      const signedBlobSidecar = ssz.deneb.SignedBlobSidecar.defaultValue();
+      signedBlobSidecar.message = blobSidecar;
+      return signedBlobSidecar;
+    });
 
-    // Full validation
-    validateBlobsSidecar(slot, beaconBlockRoot, kzgCommitments, blobsSidecar);
+    expect(signedBlobSidecars.length).to.equal(2);
 
-    // Gossip validation
-    validateGossipBlobsSidecar(signedBeaconBlock, blobsSidecar);
+    // TODO DENEB: add full validation
   });
 });