feat: update deneb types and spec to 1.4.0-alpha.1 (#5605)

* feat:  update deneb types and spec to 1.4.0-alpha.1

update config commit

* fix e2e

packages/beacon-node/test/e2e/api/impl/config.test.ts

@@ -12,6 +12,8 @@ const CONSTANT_NAMES_SKIP_LIST = new Set([
   // TODO DENEB: This constant was added then removed on a spec re-write.
   // When developing DENEB branch the tracked version still doesn't have released the removal
   "DOMAIN_BLOB_SIDECAR",
+  // TODO DENEB: Configure the blob subnets in a followup PR
+  "BLOB_SIDECAR_SUBNET_COUNT",
 ]);
 
 describe("api / impl / config", function () {

packages/beacon-node/test/spec/presets/index.test.ts

@@ -30,6 +30,7 @@ import {transition} from "./transition.js";
 // ],
 // ```
 const skipOpts: SkipOpts = {
+  skippedForks: ["eip6110"],
   // TODO: capella
   // BeaconBlockBody proof in lightclient is the new addition in v1.3.0-rc.2-hotfix
   // Skip them for now to enable subsequently

packages/beacon-node/test/spec/presets/operations.ts

@@ -66,15 +66,12 @@ const operationFns: Record<string, BlockProcessFn<CachedBeaconStateAllForks>> =
     blockFns.processVoluntaryExit(state, testCase.voluntary_exit);
   },
 
-  execution_payload: (
-    state,
-    testCase: {execution_payload: bellatrix.ExecutionPayload; execution: {execution_valid: boolean}}
-  ) => {
+  execution_payload: (state, testCase: {body: bellatrix.BeaconBlockBody; execution: {execution_valid: boolean}}) => {
     const fork = state.config.getForkSeq(state.slot);
     blockFns.processExecutionPayload(
       fork,
       state as CachedBeaconStateAllForks as CachedBeaconStateBellatrix,
-      testCase.execution_payload,
+      testCase.body,
       {
         executionPayloadStatus: testCase.execution.execution_valid
           ? ExecutionPayloadStatus.valid
@@ -127,6 +124,7 @@ export const operations: TestRunnerFn<OperationsTestCase, BeaconStateAllForks> =
         attestation: ssz.phase0.Attestation,
         attester_slashing: ssz.phase0.AttesterSlashing,
         block: ssz[fork].BeaconBlock,
+        body: ssz[fork].BeaconBlockBody,
         deposit: ssz.phase0.Deposit,
         proposer_slashing: ssz.phase0.ProposerSlashing,
         voluntary_exit: ssz.phase0.SignedVoluntaryExit,

packages/beacon-node/test/spec/specTestVersioning.ts

@@ -15,7 +15,7 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 export const ethereumConsensusSpecsTests: DownloadTestsOptions = {
-  specVersion: "v1.3.0",
+  specVersion: "v1.4.0-alpha.1",
   // Target directory is the host package root: 'packages/*/spec-tests'
   outputDir: path.join(__dirname, "../../spec-tests"),
   specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests",

packages/light-client/src/spec/utils.ts

@@ -98,6 +98,8 @@ export function upgradeLightClientHeader(
 
     // eslint-disable-next-line no-fallthrough
     case ForkName.deneb:
+      (upgradedHeader as deneb.LightClientHeader).execution.dataGasUsed =
+        ssz.deneb.LightClientHeader.fields.execution.fields.dataGasUsed.defaultValue();
       (upgradedHeader as deneb.LightClientHeader).execution.excessDataGas =
         ssz.deneb.LightClientHeader.fields.execution.fields.excessDataGas.defaultValue();
 
@@ -127,8 +129,10 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: allFor
 
   if (epoch < config.DENEB_FORK_EPOCH) {
     if (
-      (header as deneb.LightClientHeader).execution.excessDataGas &&
-      (header as deneb.LightClientHeader).execution.excessDataGas !== BigInt(0)
+      ((header as deneb.LightClientHeader).execution.dataGasUsed &&
+        (header as deneb.LightClientHeader).execution.dataGasUsed !== BigInt(0)) ||
+      ((header as deneb.LightClientHeader).execution.excessDataGas &&
+        (header as deneb.LightClientHeader).execution.excessDataGas !== BigInt(0))
     ) {
       return false;
     }

packages/light-client/test/unit/isValidLightClientHeader.test.ts

@@ -76,7 +76,7 @@ describe("isValidLightClientHeader", function () {
 
   const capellaUpgradedDenebHeader = {
     beacon: capellaLCHeader.beacon,
-    execution: {...capellaLCHeader.execution, excessDataGas: 0},
+    execution: {...capellaLCHeader.execution, dataGasUsed: 0, excessDataGas: 0},
     executionBranch: capellaLCHeader.executionBranch,
   };
 

packages/params/src/index.ts

@@ -98,6 +98,7 @@ export const {
   MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP,
 
   FIELD_ELEMENTS_PER_BLOB,
+  MAX_BLOB_COMMITMENTS_PER_BLOCK,
   MAX_BLOBS_PER_BLOCK,
 } = activePreset;
 

packages/params/src/presets/mainnet.ts

@@ -113,5 +113,6 @@ export const mainnetPreset: BeaconPreset = {
   ///////////
   // https://github.com/ethereum/consensus-specs/blob/dev/presets/mainnet/eip4844.yaml
   FIELD_ELEMENTS_PER_BLOB: 4096,
+  MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096,
   MAX_BLOBS_PER_BLOCK: 4,
 };

packages/params/src/presets/minimal.ts

@@ -120,5 +120,6 @@ export const minimalPreset: BeaconPreset = {
   ///////////
   // https://github.com/ethereum/consensus-specs/blob/dev/presets/minimal/eip4844.yaml
   FIELD_ELEMENTS_PER_BLOB: 4,
+  MAX_BLOB_COMMITMENTS_PER_BLOCK: 16,
   MAX_BLOBS_PER_BLOCK: 4,
 };

packages/params/src/types.ts

@@ -79,6 +79,7 @@ export type BeaconPreset = {
   // DENEB
   ///////////
   FIELD_ELEMENTS_PER_BLOB: number;
+  MAX_BLOB_COMMITMENTS_PER_BLOCK: number;
   MAX_BLOBS_PER_BLOCK: number;
 };
 
@@ -162,6 +163,7 @@ export const beaconPresetTypes: BeaconPresetTypes = {
   // DENEB
   ///////////
   FIELD_ELEMENTS_PER_BLOB: "number",
+  MAX_BLOB_COMMITMENTS_PER_BLOCK: "number",
   MAX_BLOBS_PER_BLOCK: "number",
 };
 

packages/params/test/e2e/ensure-config-is-synced.test.ts

@@ -8,7 +8,7 @@ import {loadConfigYaml} from "../yaml.js";
 // Not e2e, but slow. Run with e2e tests
 
 /** https://github.com/ethereum/consensus-specs/releases */
-const specConfigCommit = "v1.3.0";
+const specConfigCommit = "v1.4.0-alpha.1";
 
 describe("Ensure config is synced", function () {
   this.timeout(60 * 1000);

packages/state-transition/src/block/index.ts

@@ -51,7 +51,7 @@ export function processBlock(
         fullOrBlindedPayload as capella.FullOrBlindedExecutionPayload
       );
     }
-    processExecutionPayload(fork, state as CachedBeaconStateBellatrix, fullOrBlindedPayload, externalData);
+    processExecutionPayload(fork, state as CachedBeaconStateBellatrix, block.body, externalData);
   }
 
   processRandao(state, block, verifySignatures);

packages/state-transition/src/block/processExecutionPayload.ts

@@ -1,17 +1,18 @@
 import {ssz, allForks, capella, deneb} from "@lodestar/types";
 import {toHexString, byteArrayEquals} from "@chainsafe/ssz";
-import {ForkSeq} from "@lodestar/params";
+import {ForkSeq, MAX_BLOBS_PER_BLOCK} from "@lodestar/params";
 import {CachedBeaconStateBellatrix, CachedBeaconStateCapella} from "../types.js";
 import {getRandaoMix} from "../util/index.js";
-import {isExecutionPayload, isMergeTransitionComplete} from "../util/execution.js";
+import {isExecutionPayload, isMergeTransitionComplete, getFullOrBlindedPayloadFromBody} from "../util/execution.js";
 import {BlockExternalData, ExecutionPayloadStatus} from "./externalData.js";
 
 export function processExecutionPayload(
   fork: ForkSeq,
   state: CachedBeaconStateBellatrix | CachedBeaconStateCapella,
-  payload: allForks.FullOrBlindedExecutionPayload,
+  body: allForks.FullOrBlindedBeaconBlockBody,
   externalData: BlockExternalData
 ): void {
+  const payload = getFullOrBlindedPayloadFromBody(body);
   // Verify consistency of the parent hash, block number, base fee per gas and gas limit
   // with respect to the previous execution payload header
   if (isMergeTransitionComplete(state)) {
@@ -43,6 +44,13 @@ export function processExecutionPayload(
     throw Error(`Invalid timestamp ${payload.timestamp} genesisTime=${state.genesisTime} slot=${state.slot}`);
   }
 
+  if (fork >= ForkSeq.deneb) {
+    const blobKzgCommitmentsLen = (body as deneb.BeaconBlockBody).blobKzgCommitments?.length ?? 0;
+    if (blobKzgCommitmentsLen > MAX_BLOBS_PER_BLOCK) {
+      throw Error(`blobKzgCommitmentsLen exceeds limit=${MAX_BLOBS_PER_BLOCK}`);
+    }
+  }
+
   // Verify the execution payload is valid
   //
   // if executionEngine is null, executionEngine.onPayload MUST be called after running processBlock to get the
@@ -101,6 +109,9 @@ export function executionPayloadToPayloadHeader(
 
   if (fork >= ForkSeq.deneb) {
     // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#process_execution_payload
+    (bellatrixPayloadFields as deneb.ExecutionPayloadHeader).dataGasUsed = (
+      payload as deneb.ExecutionPayloadHeader | deneb.ExecutionPayload
+    ).dataGasUsed;
     (bellatrixPayloadFields as deneb.ExecutionPayloadHeader).excessDataGas = (
       payload as deneb.ExecutionPayloadHeader | deneb.ExecutionPayload
     ).excessDataGas;

packages/state-transition/src/slot/upgradeStateToDeneb.ts

@@ -20,14 +20,21 @@ export function upgradeStateToDeneb(stateCapella: CachedBeaconStateCapella): Cac
     epoch: stateCapella.epochCtx.epoch,
   });
 
-  // The field order of deneb latestExecutionPayloadHeader is not the same to capella
-  // all fields after excessDataGas need to explicitly set
-  stateDeneb.latestExecutionPayloadHeader.excessDataGas = ssz.UintBn256.defaultValue();
-  stateDeneb.latestExecutionPayloadHeader.blockHash = stateCapella.latestExecutionPayloadHeader.blockHash;
-  stateDeneb.latestExecutionPayloadHeader.transactionsRoot = stateCapella.latestExecutionPayloadHeader.transactionsRoot;
-  stateDeneb.latestExecutionPayloadHeader.withdrawalsRoot = stateCapella.latestExecutionPayloadHeader.withdrawalsRoot;
+  // Since excessDataGas and dataGasUsed are appened in the end to latestExecutionPayloadHeader so they should
+  // be set to defaults and need no assigning, but right now any access to latestExecutionPayloadHeader fails
+  // with LeafNode has no left node. Weirdly its beacuse of addition of the second field as with one field
+  // it seems to work.
+  //
+  // TODO DENEB: Debug and remove the following cloning
+  stateDeneb.latestExecutionPayloadHeader = ssz.deneb.BeaconState.fields.latestExecutionPayloadHeader.toViewDU({
+    ...stateCapella.latestExecutionPayloadHeader.toValue(),
+    excessDataGas: BigInt(0),
+    dataGasUsed: BigInt(0),
+  });
 
   stateDeneb.commit();
+  // Clear cache to ensure the cache of capella fields is not used by new deneb fields
+  stateDeneb["clearCache"]();
 
   return stateDeneb;
 }

packages/state-transition/src/util/execution.ts

@@ -1,4 +1,4 @@
-import {allForks, bellatrix, capella, isBlindedBeaconBlock, ssz} from "@lodestar/types";
+import {allForks, bellatrix, capella, isBlindedBeaconBlockBody, ssz} from "@lodestar/types";
 import {
   BeaconStateBellatrix,
   BeaconStateCapella,
@@ -93,10 +93,16 @@ export function isExecutionBlockBodyType(
 export function getFullOrBlindedPayload(
   block: allForks.FullOrBlindedBeaconBlock
 ): allForks.FullOrBlindedExecutionPayload {
-  if (isBlindedBeaconBlock(block)) {
-    return block.body.executionPayloadHeader;
-  } else if ((block as bellatrix.BeaconBlock).body.executionPayload !== undefined) {
-    return (block as bellatrix.BeaconBlock).body.executionPayload;
+  return getFullOrBlindedPayloadFromBody(block.body);
+}
+
+export function getFullOrBlindedPayloadFromBody(
+  body: allForks.FullOrBlindedBeaconBlockBody
+): allForks.FullOrBlindedExecutionPayload {
+  if (isBlindedBeaconBlockBody(body)) {
+    return body.executionPayloadHeader;
+  } else if ((body as bellatrix.BeaconBlockBody).executionPayload !== undefined) {
+    return (body as bellatrix.BeaconBlockBody).executionPayload;
   } else {
     throw Error("Ǹot allForks.FullOrBlindedBeaconBlock");
   }

packages/state-transition/test/unit/upgradeState.test.ts

@@ -0,0 +1,58 @@
+import {expect} from "chai";
+import {ssz} from "@lodestar/types";
+import {ForkName} from "@lodestar/params";
+import {createCachedBeaconState, PubkeyIndexMap} from "@lodestar/state-transition";
+import {createBeaconConfig, ChainForkConfig, createChainForkConfig} from "@lodestar/config";
+import {config as chainConfig} from "@lodestar/config/default";
+
+import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js";
+
+describe("upgradeState", () => {
+  it("upgradeStateToDeneb", () => {
+    const capellaState = ssz.capella.BeaconState.defaultViewDU();
+    const config = getConfig(ForkName.capella);
+    const stateView = createCachedBeaconState(
+      capellaState,
+      {
+        config: createBeaconConfig(config, capellaState.genesisValidatorsRoot),
+        pubkey2index: new PubkeyIndexMap(),
+        index2pubkey: [],
+      },
+      {skipSyncCommitteeCache: true}
+    );
+    const newState = upgradeStateToDeneb(stateView);
+    expect(() => newState.toValue()).to.not.throw();
+  });
+});
+
+const ZERO_HASH = Buffer.alloc(32, 0);
+/** default config with ZERO_HASH as genesisValidatorsRoot */
+const config = createBeaconConfig(chainConfig, ZERO_HASH);
+
+/* eslint-disable @typescript-eslint/naming-convention */
+function getConfig(fork: ForkName, forkEpoch = 0): ChainForkConfig {
+  switch (fork) {
+    case ForkName.phase0:
+      return config;
+    case ForkName.altair:
+      return createChainForkConfig({ALTAIR_FORK_EPOCH: forkEpoch});
+    case ForkName.bellatrix:
+      return createChainForkConfig({
+        ALTAIR_FORK_EPOCH: 0,
+        BELLATRIX_FORK_EPOCH: forkEpoch,
+      });
+    case ForkName.capella:
+      return createChainForkConfig({
+        ALTAIR_FORK_EPOCH: 0,
+        BELLATRIX_FORK_EPOCH: 0,
+        CAPELLA_FORK_EPOCH: forkEpoch,
+      });
+    case ForkName.deneb:
+      return createChainForkConfig({
+        ALTAIR_FORK_EPOCH: 0,
+        BELLATRIX_FORK_EPOCH: 0,
+        CAPELLA_FORK_EPOCH: 0,
+        DENEB_FORK_EPOCH: forkEpoch,
+      });
+  }
+}

packages/types/src/deneb/sszTypes.ts

@@ -1,6 +1,7 @@
 import {ContainerType, ListCompositeType, ByteVectorType, VectorCompositeType} from "@chainsafe/ssz";
 import {
   HISTORICAL_ROOTS_LIMIT,
+  MAX_BLOB_COMMITMENTS_PER_BLOCK,
   FIELD_ELEMENTS_PER_BLOB,
   MAX_BLOBS_PER_BLOCK,
   MAX_REQUEST_BLOCKS,
@@ -20,6 +21,7 @@ const {
   Slot,
   Root,
   BLSSignature,
+  UintBn64,
   UintBn256,
   Bytes32,
   Bytes48,
@@ -50,7 +52,7 @@ export const Blobs = new ListCompositeType(Blob, MAX_BLOBS_PER_BLOCK);
 export const BlindedBlob = Bytes32;
 export const BlindedBlobs = new ListCompositeType(BlindedBlob, MAX_BLOBS_PER_BLOCK);
 export const VersionedHash = Bytes32;
-export const BlobKzgCommitments = new ListCompositeType(KZGCommitment, MAX_BLOBS_PER_BLOCK);
+export const BlobKzgCommitments = new ListCompositeType(KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK);
 
 // Constants
 
@@ -121,15 +123,17 @@ export const BeaconBlockAndBlobsSidecarByRootRequest = new ListCompositeType(Roo
 export const ExecutionPayload = new ContainerType(
   {
     ...capellaSsz.ExecutionPayload.fields,
-    excessDataGas: UintBn256, // New in DENEB
+    dataGasUsed: UintBn64, // New in DENEB
+    excessDataGas: UintBn64, // New in DENEB
   },
   {typeName: "ExecutionPayload", jsonCase: "eth2"}
 );
 
 export const ExecutionPayloadHeader = new ContainerType(
   {
     ...capellaSsz.ExecutionPayloadHeader.fields,
-    excessDataGas: UintBn256, // New in DENEB
+    dataGasUsed: UintBn64, // New in DENEB
+    excessDataGas: UintBn64, // New in DENEB
   },
   {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"}
 );
@@ -233,8 +237,9 @@ export const SignedBeaconBlockAndBlobsSidecar = new ContainerType(
 
 export const BlindedBeaconBlockBody = new ContainerType(
   {
-    ...BeaconBlockBody.fields,
+    ...altairSsz.BeaconBlockBody.fields,
     executionPayloadHeader: ExecutionPayloadHeader, // Modified in DENEB
+    blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges,
     blobKzgCommitments: BlobKzgCommitments, // New in DENEB
   },
   {typeName: "BlindedBeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true}

packages/types/src/utils/typeguards.ts

@@ -1,10 +1,13 @@
 import {
   FullOrBlindedBeaconBlock,
   FullOrBlindedSignedBeaconBlock,
+  FullOrBlindedBeaconBlockBody,
   FullOrBlindedExecutionPayload,
   ExecutionPayloadHeader,
   FullOrBlindedBlobSidecar,
   FullOrBlindedSignedBlobSidecar,
+  BlindedBeaconBlockBody,
+  BlindedBeaconBlock,
 } from "../allForks/types.js";
 import {ts as bellatrix} from "../bellatrix/index.js";
 import {ts as deneb} from "../deneb/index.js";
@@ -15,8 +18,12 @@ export function isBlindedExecution(payload: FullOrBlindedExecutionPayload): payl
   return (payload as ExecutionPayloadHeader).transactionsRoot !== undefined;
 }
 
-export function isBlindedBeaconBlock(block: FullOrBlindedBeaconBlock): block is bellatrix.BlindedBeaconBlock {
-  return (block as bellatrix.BlindedBeaconBlock).body.executionPayloadHeader !== undefined;
+export function isBlindedBeaconBlock(block: FullOrBlindedBeaconBlock): block is BlindedBeaconBlock {
+  return isBlindedBeaconBlockBody(block.body);
+}
+
+export function isBlindedBeaconBlockBody(body: FullOrBlindedBeaconBlockBody): body is BlindedBeaconBlockBody {
+  return (body as BlindedBeaconBlockBody).executionPayloadHeader !== undefined;
 }
 
 export function isBlindedSignedBeaconBlock(

packages/validator/src/util/params.ts

@@ -208,5 +208,6 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record<keyof ConfigWit
     /////////////////
     FIELD_ELEMENTS_PER_BLOB: denebForkRelevant,
     MAX_BLOBS_PER_BLOCK: denebForkRelevant,
+    MAX_BLOB_COMMITMENTS_PER_BLOCK: denebForkRelevant,
   };
 }